60#include "llvm/IR/IntrinsicsAArch64.h"
95#define DEBUG_TYPE "aarch64-lower"
98STATISTIC(NumShiftInserts,
"Number of vector shift inserts");
99STATISTIC(NumOptimizedImms,
"Number of times immediates were optimized");
106 cl::desc(
"Allow AArch64 Local Dynamic TLS code generation"),
111 cl::desc(
"Enable AArch64 logical imm instruction "
121 cl::desc(
"Combine extends of AArch64 masked "
122 "gather intrinsics"),
154 switch (EC.getKnownMinValue()) {
170 "Expected scalable predicate vector type!");
192 "Expected legal vector type!");
280 if (Subtarget->
hasSVE()) {
308 if (useSVEForFixedLengthVectorVT(VT))
312 if (useSVEForFixedLengthVectorVT(VT))
730#define LCALLNAMES(A, B, N) \
731 setLibcallName(A##N##_RELAX, #B #N "_relax"); \
732 setLibcallName(A##N##_ACQ, #B #N "_acq"); \
733 setLibcallName(A##N##_REL, #B #N "_rel"); \
734 setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
735#define LCALLNAME4(A, B) \
736 LCALLNAMES(A, B, 1) \
737 LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
738#define LCALLNAME5(A, B) \
739 LCALLNAMES(A, B, 1) \
740 LCALLNAMES(A, B, 2) \
741 LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
742 LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
743 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
744 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
745 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
746 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
747 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
1092 if (Subtarget->
hasSVE()) {
1203 if (useSVEForFixedLengthVectorVT(VT))
1204 addTypeForFixedLengthSVE(VT);
1206 if (useSVEForFixedLengthVectorVT(VT))
1207 addTypeForFixedLengthSVE(VT);
1280void AArch64TargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedBitwiseVT) {
1347 for (
unsigned Opcode :
1360void AArch64TargetLowering::addTypeForFixedLengthSVE(
MVT VT) {
1431void AArch64TargetLowering::addDRTypeForNEON(
MVT VT) {
1436void AArch64TargetLowering::addQRTypeForNEON(
MVT VT) {
1451 const APInt &Demanded,
1454 uint64_t OldImm = Imm, NewImm, Enc;
1455 uint64_t Mask = ((uint64_t)(-1LL) >> (64 -
Size)), OrigMask = Mask;
1459 if (Imm == 0 || Imm == Mask ||
1463 unsigned EltSize =
Size;
1479 uint64_t RotatedImm =
1480 ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
1482 uint64_t Sum = RotatedImm + NonDemandedBits;
1483 bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
1484 uint64_t Ones = (Sum + Carry) & NonDemandedBits;
1485 NewImm = (Imm | Ones) & Mask;
1499 uint64_t Hi = Imm >> EltSize, DemandedBitsHi =
DemandedBits >> EltSize;
1502 if (((Imm ^ Hi) & (
DemandedBits & DemandedBitsHi) & Mask) != 0)
1513 while (EltSize <
Size) {
1514 NewImm |= NewImm << EltSize;
1520 "demanded bits should never be altered");
1521 assert(OldImm != NewImm &&
"the new imm shouldn't be equal to the old imm");
1524 EVT VT = Op.getValueType();
1530 if (NewImm == 0 || NewImm == OrigMask) {
1531 New = TLO.
DAG.
getNode(Op.getOpcode(),
DL, VT, Op.getOperand(0),
1555 EVT VT = Op.getValueType();
1561 "i32 or i64 is expected after legalization.");
1568 switch (Op.getOpcode()) {
1572 NewOpc =
Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1575 NewOpc =
Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1578 NewOpc =
Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1584 uint64_t Imm =
C->getZExtValue();
1593 switch (Op.getOpcode()) {
1616 case Intrinsic::aarch64_ldaxr:
1617 case Intrinsic::aarch64_ldxr: {
1633 case Intrinsic::aarch64_neon_umaxv:
1634 case Intrinsic::aarch64_neon_uminv: {
1639 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1696 Ty.getSizeInBytes() != 16 ||
1719#define MAKE_CASE(V) \
2021 Register DestReg =
MI.getOperand(0).getReg();
2022 Register IfTrueReg =
MI.getOperand(1).getReg();
2023 Register IfFalseReg =
MI.getOperand(2).getReg();
2024 unsigned CondCode =
MI.getOperand(3).getImm();
2025 bool NZCVKilled =
MI.getOperand(4).isKill();
2056 MI.eraseFromParent();
2064 "SEH does not use catchret!");
2070 switch (
MI.getOpcode()) {
2077 case AArch64::F128CSEL:
2080 case TargetOpcode::STACKMAP:
2081 case TargetOpcode::PATCHPOINT:
2082 case TargetOpcode::STATEPOINT:
2085 case AArch64::CATCHRET:
2247 CondCode, CondCode2);
2254 bool IsLegal = (
C >> 12 == 0) || ((
C & 0xFFFULL) == 0 &&
C >> 24 == 0);
2256 <<
" legal: " << (IsLegal ?
"yes\n" :
"no\n"));
2278 EVT VT = LHS.getValueType();
2280 assert(VT !=
MVT::f16 &&
"Lowering of strict fp16 not yet implemented");
2288 EVT VT = LHS.getValueType();
2289 const bool FullFP16 =
2308 if (
isCMN(RHS, CC)) {
2311 RHS = RHS.getOperand(1);
2312 }
else if (
isCMN(LHS, CC)) {
2316 LHS = LHS.getOperand(1);
2331 return LHS.getValue(1);
2397 unsigned Opcode = 0;
2398 const bool FullFP16 =
2401 if (LHS.getValueType().isFloatingPoint()) {
2403 if (LHS.getValueType() ==
MVT::f16 && !FullFP16) {
2408 }
else if (RHS.getOpcode() ==
ISD::SUB) {
2413 RHS = RHS.getOperand(1);
2423 return DAG.
getNode(Opcode,
DL,
MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
2441 bool &MustBeFirst,
bool WillNegate,
2442 unsigned Depth = 0) {
2450 MustBeFirst =
false;
2457 bool IsOR = Opcode ==
ISD::OR;
2469 if (MustBeFirstL && MustBeFirstR)
2475 if (!CanNegateL && !CanNegateR)
2479 CanNegate = WillNegate && CanNegateL && CanNegateR;
2482 MustBeFirst = !CanNegate;
2487 MustBeFirst = MustBeFirstL || MustBeFirstR;
2513 CC = getSetCCInverse(CC, LHS.getValueType());
2519 assert(LHS.getValueType().isFloatingPoint());
2532 Predicate = ExtraCC;
2545 bool IsOR = Opcode ==
ISD::OR;
2551 assert(ValidL &&
"Valid conjunction/disjunction tree");
2558 assert(ValidR &&
"Valid conjunction/disjunction tree");
2563 assert(!MustBeFirstR &&
"Valid conjunction/disjunction tree");
2572 bool NegateAfterAll;
2576 assert(CanNegateR &&
"at least one side must be negatable");
2577 assert(!MustBeFirstR &&
"invalid conjunction/disjunction tree");
2581 NegateAfterR =
true;
2584 NegateR = CanNegateR;
2585 NegateAfterR = !CanNegateR;
2588 NegateAfterAll = !Negate;
2590 assert(Opcode ==
ISD::AND &&
"Valid conjunction/disjunction tree");
2591 assert(!Negate &&
"Valid conjunction/disjunction tree");
2595 NegateAfterR =
false;
2596 NegateAfterAll =
false;
2616 bool DummyCanNegate;
2617 bool DummyMustBeFirst;
2629 auto isSupportedExtend = [&](
SDValue V) {
2635 uint64_t Mask = MaskCst->getZExtValue();
2636 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2642 if (!Op.hasOneUse())
2645 if (isSupportedExtend(Op))
2648 unsigned Opc = Op.getOpcode();
2651 uint64_t Shift = ShiftCst->getZExtValue();
2652 if (isSupportedExtend(Op.getOperand(0)))
2653 return (Shift <= 4) ? 2 : 1;
2654 EVT VT = Op.getValueType();
2666 EVT VT = RHS.getValueType();
2667 uint64_t
C = RHSC->getZExtValue();
2675 if ((VT ==
MVT::i32 &&
C != 0x80000000 &&
2677 (VT ==
MVT::i64 &&
C != 0x80000000ULL &&
2696 if ((VT ==
MVT::i32 &&
C != INT32_MAX &&
2707 if ((VT ==
MVT::i32 &&
C != UINT32_MAX &&
2732 SDValue TheLHS =
isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2764 LHS.getNode()->hasNUsesOfValue(1, 0)) {
2771 RHS.getValueType()),
2793static std::pair<SDValue, SDValue>
2796 "Unsupported value type");
2799 SDValue LHS = Op.getOperand(0);
2800 SDValue RHS = Op.getOperand(1);
2802 switch (Op.getOpcode()) {
2825 bool IsSigned = Op.getOpcode() ==
ISD::SMULO;
2826 if (Op.getValueType() ==
MVT::i32) {
2874 assert(Op.getValueType() ==
MVT::i64 &&
"Expected an i64 value type");
2903 Overflow =
Value.getValue(1);
2905 return std::make_pair(
Value, Overflow);
2909 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
2910 return LowerToScalableOp(Op, DAG);
2964 if (!CFVal || !CTVal)
2992 EVT VT = Op.getValueType();
3001 bool ExtraOp =
false;
3002 switch (Op.getOpcode()) {
3022 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3023 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3064 bool IsStream = !Locality;
3068 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
3072 Locality = 3 - Locality;
3076 unsigned PrfOp = (IsWrite << 4) |
3086 if (
Op.getValueType().isScalableVector())
3095 if (
Op.getValueType().isScalableVector())
3098 bool IsStrict =
Op->isStrictFPOpcode();
3099 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3104 if (useSVEForFixedLengthVectorVT(SrcVT))
3119 EVT InVT =
Op.getOperand(0).getValueType();
3120 EVT VT =
Op.getValueType();
3126 return LowerToPredicatedOp(Op, DAG, Opcode);
3137 Op.getOpcode(), dl,
Op.getValueType(),
3143 if (VTSize < InVTSize) {
3151 if (VTSize > InVTSize) {
3157 return DAG.
getNode(
Op.getOpcode(), dl, VT, Ext);
3166 bool IsStrict =
Op->isStrictFPOpcode();
3167 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3170 return LowerVectorFP_TO_INT(Op, DAG);
3174 assert(!IsStrict &&
"Lowering of strict fp16 not yet implemented");
3177 Op.getOpcode(), dl,
Op.getValueType(),
3194 EVT VT =
Op.getValueType();
3197 EVT InVT =
In.getValueType();
3198 unsigned Opc =
Op.getOpcode();
3206 In = DAG.
getNode(CastOpc, dl, CastVT, In);
3207 return DAG.
getNode(Opc, dl, VT, In);
3212 return LowerToPredicatedOp(Op, DAG, Opcode);
3217 if (VTSize < InVTSize) {
3225 if (VTSize > InVTSize) {
3228 In = DAG.
getNode(CastOpc, dl, CastVT, In);
3229 return DAG.
getNode(Opc, dl, VT, In);
3237 if (
Op.getValueType().isVector())
3238 return LowerVectorINT_TO_FP(Op, DAG);
3240 bool IsStrict =
Op->isStrictFPOpcode();
3241 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3246 assert(!IsStrict &&
"Lowering of strict fp16 not yet implemented");
3271 EVT ArgVT =
Arg.getValueType();
3279 Entry.IsSExt =
false;
3280 Entry.IsZExt =
false;
3281 Args.push_back(Entry);
3284 : RTLIB::SINCOS_STRET_F32;
3295 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3296 return CallResult.first;
3300 EVT OpVT = Op.getValueType();
3322 switch (OrigSimpleTy) {
3335 unsigned ExtOpcode) {
3351 EVT VT =
N->getValueType(0);
3359 unsigned HalfSize = EltSize / 2;
3361 if (!
isIntN(HalfSize,
C->getSExtValue()))
3364 if (!
isUIntN(HalfSize,
C->getZExtValue()))
3384 EVT VT =
N->getValueType(0);
3390 for (
unsigned i = 0; i != NumElts; ++i) {
3392 const APInt &CInt =
C->getAPIntValue();
3413 unsigned Opcode =
N->getOpcode();
3424 unsigned Opcode =
N->getOpcode();
3458 EVT VT =
Op.getValueType();
3463 if (VT.
isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
3469 "unexpected type for custom-lowering ISD::MUL");
3470 SDNode *N0 =
Op.getOperand(0).getNode();
3471 SDNode *N1 =
Op.getOperand(1).getNode();
3472 unsigned NewOpc = 0;
3476 if (isN0SExt && isN1SExt)
3481 if (isN0ZExt && isN1ZExt)
3483 else if (isN1SExt || isN1ZExt) {
3517 "unexpected types for extended operands to VMULL");
3518 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
3539SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(
SDValue Op,
3549 case Intrinsic::aarch64_neon_abs: {
3550 EVT Ty =
Op.getValueType();
3556 }
else if (Ty.isVector() && Ty.isInteger() &&
isTypeLegal(Ty)) {
3562 case Intrinsic::aarch64_neon_smax:
3564 Op.getOperand(1),
Op.getOperand(2));
3565 case Intrinsic::aarch64_neon_umax:
3567 Op.getOperand(1),
Op.getOperand(2));
3568 case Intrinsic::aarch64_neon_smin:
3570 Op.getOperand(1),
Op.getOperand(2));
3571 case Intrinsic::aarch64_neon_umin:
3573 Op.getOperand(1),
Op.getOperand(2));
3575 case Intrinsic::aarch64_sve_sunpkhi:
3578 case Intrinsic::aarch64_sve_sunpklo:
3581 case Intrinsic::aarch64_sve_uunpkhi:
3584 case Intrinsic::aarch64_sve_uunpklo:
3587 case Intrinsic::aarch64_sve_clasta_n:
3589 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3590 case Intrinsic::aarch64_sve_clastb_n:
3592 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3593 case Intrinsic::aarch64_sve_lasta:
3595 Op.getOperand(1),
Op.getOperand(2));
3596 case Intrinsic::aarch64_sve_lastb:
3598 Op.getOperand(1),
Op.getOperand(2));
3599 case Intrinsic::aarch64_sve_rev:
3602 case Intrinsic::aarch64_sve_tbl:
3604 Op.getOperand(1),
Op.getOperand(2));
3605 case Intrinsic::aarch64_sve_trn1:
3607 Op.getOperand(1),
Op.getOperand(2));
3608 case Intrinsic::aarch64_sve_trn2:
3610 Op.getOperand(1),
Op.getOperand(2));
3611 case Intrinsic::aarch64_sve_uzp1:
3613 Op.getOperand(1),
Op.getOperand(2));
3614 case Intrinsic::aarch64_sve_uzp2:
3616 Op.getOperand(1),
Op.getOperand(2));
3617 case Intrinsic::aarch64_sve_zip1:
3619 Op.getOperand(1),
Op.getOperand(2));
3620 case Intrinsic::aarch64_sve_zip2:
3622 Op.getOperand(1),
Op.getOperand(2));
3623 case Intrinsic::aarch64_sve_ptrue:
3626 case Intrinsic::aarch64_sve_clz:
3628 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3629 case Intrinsic::aarch64_sve_cnt: {
3632 if (
Data.getValueType().isFloatingPoint())
3635 Op.getOperand(2),
Data,
Op.getOperand(1));
3637 case Intrinsic::aarch64_sve_dupq_lane:
3638 return LowerDUPQLane(Op, DAG);
3639 case Intrinsic::aarch64_sve_convert_from_svbool:
3642 case Intrinsic::aarch64_sve_fneg:
3644 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3645 case Intrinsic::aarch64_sve_frintp:
3647 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3648 case Intrinsic::aarch64_sve_frintm:
3650 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3651 case Intrinsic::aarch64_sve_frinti:
3653 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3654 case Intrinsic::aarch64_sve_frintx:
3656 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3657 case Intrinsic::aarch64_sve_frinta:
3659 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3660 case Intrinsic::aarch64_sve_frintn:
3662 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3663 case Intrinsic::aarch64_sve_frintz:
3665 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3666 case Intrinsic::aarch64_sve_ucvtf:
3668 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3670 case Intrinsic::aarch64_sve_scvtf:
3672 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3674 case Intrinsic::aarch64_sve_fcvtzu:
3676 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3678 case Intrinsic::aarch64_sve_fcvtzs:
3680 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3682 case Intrinsic::aarch64_sve_fsqrt:
3684 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3685 case Intrinsic::aarch64_sve_frecpx:
3687 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3688 case Intrinsic::aarch64_sve_fabs:
3690 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3691 case Intrinsic::aarch64_sve_abs:
3693 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3694 case Intrinsic::aarch64_sve_neg:
3696 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3697 case Intrinsic::aarch64_sve_convert_to_svbool: {
3698 EVT OutVT =
Op.getValueType();
3699 EVT InVT =
Op.getOperand(1).getValueType();
3703 return Op.getOperand(1);
3710 return DAG.
getNode(
ISD::AND, dl, OutVT, Reinterpret, MaskReinterpret);
3713 case Intrinsic::aarch64_sve_insr: {
3720 Op.getOperand(1), Scalar);
3722 case Intrinsic::aarch64_sve_rbit:
3724 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3726 case Intrinsic::aarch64_sve_revb:
3728 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3729 case Intrinsic::aarch64_sve_sxtb:
3732 Op.getOperand(2),
Op.getOperand(3),
3735 case Intrinsic::aarch64_sve_sxth:
3738 Op.getOperand(2),
Op.getOperand(3),
3741 case Intrinsic::aarch64_sve_sxtw:
3744 Op.getOperand(2),
Op.getOperand(3),
3747 case Intrinsic::aarch64_sve_uxtb:
3750 Op.getOperand(2),
Op.getOperand(3),
3753 case Intrinsic::aarch64_sve_uxth:
3756 Op.getOperand(2),
Op.getOperand(3),
3759 case Intrinsic::aarch64_sve_uxtw:
3762 Op.getOperand(2),
Op.getOperand(3),
3769 unsigned Reg =
RegInfo->getLocalAddressRegister(MF);
3771 Op.getSimpleValueType());
3779 SDValue IncomingFPOp =
Op.getOperand(2);
3784 "llvm.eh.recoverfp must take a function as the first argument");
3785 return IncomingFPOp;
3788 case Intrinsic::aarch64_neon_vsri:
3789 case Intrinsic::aarch64_neon_vsli: {
3790 EVT Ty =
Op.getValueType();
3795 assert(
Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
3797 bool IsShiftRight = IntNo == Intrinsic::aarch64_neon_vsri;
3799 return DAG.
getNode(Opcode, dl, Ty,
Op.getOperand(1),
Op.getOperand(2),
3803 case Intrinsic::aarch64_neon_srhadd:
3804 case Intrinsic::aarch64_neon_urhadd:
3805 case Intrinsic::aarch64_neon_shadd:
3806 case Intrinsic::aarch64_neon_uhadd: {
3807 bool IsSignedAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3808 IntNo == Intrinsic::aarch64_neon_shadd);
3809 bool IsRoundingAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3810 IntNo == Intrinsic::aarch64_neon_urhadd);
3813 : (IsRoundingAdd ? AArch64ISD::
URHADD : AArch64ISD::
UHADD);
3814 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1),
3818 case Intrinsic::aarch64_neon_uabd: {
3820 Op.getOperand(1),
Op.getOperand(2));
3822 case Intrinsic::aarch64_neon_sabd: {
3824 Op.getOperand(1),
Op.getOperand(2));
3829bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(
EVT VT)
const {
3837bool AArch64TargetLowering::isVectorLoadExtDesirable(
SDValue ExtVal)
const {
3842 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
3843 {std::make_tuple(
false,
false,
false),
3845 {std::make_tuple(
false,
false,
true),
3847 {std::make_tuple(
false,
true,
false),
3849 {std::make_tuple(
false,
true,
true),
3851 {std::make_tuple(
true,
false,
false),
3853 {std::make_tuple(
true,
false,
true),
3855 {std::make_tuple(
true,
true,
false),
3857 {std::make_tuple(
true,
true,
true),
3860 auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
3861 return AddrModes.find(Key)->second;
3865 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
3866 {std::make_tuple(
false,
false,
false),
3868 {std::make_tuple(
false,
false,
true),
3870 {std::make_tuple(
false,
true,
false),
3872 {std::make_tuple(
false,
true,
true),
3874 {std::make_tuple(
true,
false,
false),
3876 {std::make_tuple(
true,
false,
true),
3878 {std::make_tuple(
true,
true,
false),
3880 {std::make_tuple(
true,
true,
true),
3883 auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
3884 return AddrModes.find(Key)->second;
3910 unsigned Opcode = Index.getOpcode();
3919 if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
3937 unsigned &Opcode,
bool IsGather,
3944 if (
auto SplatVal = DAG.
getSplatValue(Index.getOperand(1))) {
3949 Index = Index->getOperand(0);
3963 uint64_t OffsetVal =
Offset->getZExtValue();
3967 if (OffsetVal % ScalarSizeInBytes || OffsetVal / ScalarSizeInBytes > 31) {
3969 BasePtr = ConstOffset;
3970 Index = Index->getOperand(0);
3976 BasePtr = Index->getOperand(0);
3977 Index = ConstOffset;
3984 assert(MGT &&
"Can only custom lower gather load nodes");
3998 bool IdxNeedsExtend =
4000 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4014 PassThru = getSVESafeBitCast(PassThruVT, PassThru, DAG);
4021 Index = Index.getOperand(0);
4027 if (ResNeedsSignExtend)
4034 SDValue Cast = getSVESafeBitCast(VT, Gather, DAG);
4045 assert(MSC &&
"Can only custom lower scatter store nodes");
4060 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4074 StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG);
4079 Index = Index.getOperand(0);
4107 {Undef, Undef, Undef, Undef});
4117 return DAG.
getStore(ST->getChain(),
DL, ExtractTrunc,
4118 ST->getBasePtr(), ST->getMemOperand());
4128 assert (StoreNode &&
"Can only custom lower store nodes");
4136 if (useSVEForFixedLengthVectorVT(VT))
4137 return LowerFixedLengthVectorStoreToSVE(Op, DAG);
4172 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4186 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4196 MVT VT =
Op.getSimpleValueType();
4218 switch (Op.getOpcode()) {
4225 return LowerGlobalAddress(Op, DAG);
4227 return LowerGlobalTLSAddress(Op, DAG);
4231 return LowerSETCC(Op, DAG);
4233 return LowerBR_CC(Op, DAG);
4235 return LowerSELECT(Op, DAG);
4237 return LowerSELECT_CC(Op, DAG);
4239 return LowerJumpTable(Op, DAG);
4241 return LowerBR_JT(Op, DAG);
4243 return LowerConstantPool(Op, DAG);
4245 return LowerBlockAddress(Op, DAG);
4247 return LowerVASTART(Op, DAG);
4249 return LowerVACOPY(Op, DAG);
4251 return LowerVAARG(Op, DAG);
4296 return LowerFP_ROUND(Op, DAG);
4298 return LowerFP_EXTEND(Op, DAG);
4300 return LowerFRAMEADDR(Op, DAG);
4302 return LowerSPONENTRY(Op, DAG);
4304 return LowerRETURNADDR(Op, DAG);
4306 return LowerADDROFRETURNADDR(Op, DAG);
4308 return LowerCONCAT_VECTORS(Op, DAG);
4310 return LowerINSERT_VECTOR_ELT(Op, DAG);
4312 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4314 return LowerBUILD_VECTOR(Op, DAG);
4316 return LowerVECTOR_SHUFFLE(Op, DAG);
4318 return LowerSPLAT_VECTOR(Op, DAG);
4320 return LowerEXTRACT_SUBVECTOR(Op, DAG);
4322 return LowerINSERT_SUBVECTOR(Op, DAG);
4325 return LowerDIV(Op, DAG);
4341 return LowerVectorSRA_SRL_SHL(Op, DAG);
4343 return LowerShiftLeftParts(Op, DAG);
4346 return LowerShiftRightParts(Op, DAG);
4348 return LowerCTPOP(Op, DAG);
4350 return LowerFCOPYSIGN(Op, DAG);
4352 return LowerVectorOR(Op, DAG);
4354 return LowerXOR(Op, DAG);
4361 return LowerINT_TO_FP(Op, DAG);
4366 return LowerFP_TO_INT(Op, DAG);
4368 return LowerFSINCOS(Op, DAG);
4370 return LowerFLT_ROUNDS_(Op, DAG);
4372 return LowerMUL(Op, DAG);
4374 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4376 return LowerSTORE(Op, DAG);
4378 return LowerMGATHER(Op, DAG);
4380 return LowerMSCATTER(Op, DAG);
4382 return LowerVECREDUCE_SEQ_FADD(Op, DAG);
4394 return LowerVECREDUCE(Op, DAG);
4396 return LowerATOMIC_LOAD_SUB(Op, DAG);
4398 return LowerATOMIC_LOAD_AND(Op, DAG);
4400 return LowerDYNAMIC_STACKALLOC(Op, DAG);
4402 return LowerVSCALE(Op, DAG);
4406 return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
4415 return LowerToPredicatedOp(Op, DAG,
4419 return LowerTRUNCATE(Op, DAG);
4421 if (useSVEForFixedLengthVectorVT(Op.getValueType()))
4422 return LowerFixedLengthVectorLoadToSVE(Op, DAG);
4427 return LowerToScalableOp(Op, DAG);
4435 return LowerFixedLengthVectorSelectToSVE(Op, DAG);
4437 return LowerABS(Op, DAG);
4447 return LowerCTTZ(Op, DAG);
4455bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
4456 EVT VT,
bool OverrideNEON)
const {
4506 bool IsVarArg)
const {
4543SDValue AArch64TargetLowering::LowerFormalArguments(
4544 SDValue Chain, CallingConv::ID CallConv,
bool isVarArg,
4563 unsigned NumArgs = Ins.size();
4565 unsigned CurArgIdx = 0;
4566 for (
unsigned i = 0; i != NumArgs; ++i) {
4567 MVT ValVT = Ins[i].VT;
4568 if (Ins[i].isOrigArg()) {
4569 std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
4570 CurArgIdx = Ins[i].getOrigArgIndex();
4585 assert(!Res &&
"Call operand has unhandled type");
4589 unsigned ExtraArgLocs = 0;
4590 for (
unsigned i = 0, e =
Ins.size(); i != e; ++i) {
4593 if (Ins[i].Flags.isByVal()) {
4597 int Size =
Ins[i].Flags.getByValSize();
4598 unsigned NumRegs = (
Size + 7) / 8;
4617 RC = &AArch64::GPR32RegClass;
4619 RC = &AArch64::GPR64RegClass;
4621 RC = &AArch64::FPR16RegClass;
4623 RC = &AArch64::FPR32RegClass;
4625 RC = &AArch64::FPR64RegClass;
4627 RC = &AArch64::FPR128RegClass;
4630 RC = &AArch64::PPRRegClass;
4632 RC = &AArch64::ZPRRegClass;
4650 "Only scalable vectors can be passed indirectly");
4674 !Ins[i].Flags.isInConsecutiveRegs())
4675 BEAlign = 8 - ArgSize;
4695 "Only scalable vectors can be passed indirectly");
4718 "Only scalable vectors can be passed indirectly");
4721 unsigned NumParts = 1;
4722 if (Ins[i].Flags.isInConsecutiveRegs()) {
4723 assert(!Ins[i].Flags.isInConsecutiveRegsLast());
4724 while (!Ins[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
4733 while (NumParts > 0) {
4744 BytesIncrement, Flags);
4767 saveVarArgRegisters(CCInfo, DAG,
DL, Chain);
4771 unsigned StackOffset = CCInfo.getNextStackOffset();
4783 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes,
4787 if (!CCInfo.isAllocated(AArch64::X8)) {
4788 unsigned X8VReg = MF.
addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
4798 for (
unsigned I = 0,
E =
Ins.size();
I !=
E; ++
I) {
4799 if (Ins[
I].Flags.isInReg()) {
4814 unsigned StackArgSize = CCInfo.getNextStackOffset();
4816 if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
4820 StackArgSize =
alignTo(StackArgSize, 16);
4840void AArch64TargetLowering::saveVarArgRegisters(
CCState &CCInfo,
4853 AArch64::X3, AArch64::X4, AArch64::X5,
4854 AArch64::X6, AArch64::X7 };
4858 unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
4860 if (GPRSaveSize != 0) {
4863 if (GPRSaveSize & 15)
4871 for (
unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
4879 (i - FirstVariadicGPR) * 8)
4891 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
4892 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
4893 static const unsigned NumFPRArgRegs =
array_lengthof(FPRArgRegs);
4896 unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
4898 if (FPRSaveSize != 0) {
4903 for (
unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
4904 unsigned VReg = MF.
addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
4919 if (!MemOps.
empty()) {
4926SDValue AArch64TargetLowering::LowerCallResult(
4927 SDValue Chain,
SDValue InFlag, CallingConv::ID CallConv,
bool isVarArg,
4940 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
4945 if (i == 0 && isThisReturn) {
4947 "unexpected return calling convention register assignment");
5006bool AArch64TargetLowering::isEligibleForTailCallOptimization(
5026 bool CCMatch = CallerCC == CalleeCC;
5041 if (i->hasByValAttr())
5050 if (i->hasInRegAttr())
5068 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
5079 "Unexpected variadic calling convention");
5082 if (isVarArg && !Outs.
empty()) {
5090 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs,
C);
5094 if (!ArgLoc.isRegLoc())
5105 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
5107 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
5109 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
5110 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
5112 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
5121 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs,
C);
5133 A.getValVT().isScalableVector()) &&
5134 "Expected value to be scalable");
5154 int ClobberedFI)
const {
5157 int64_t LastByte = FirstByte + MFI.
getObjectSize(ClobberedFI) - 1;
5170 if (FI->getIndex() < 0) {
5172 int64_t InLastByte = InFirstByte;
5175 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
5176 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
5184bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
5185 bool TailCallOpt)
const {
5192AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
5201 bool &IsTailCall = CLI.IsTailCall;
5202 CallingConv::ID CallConv = CLI.CallConv;
5203 bool IsVarArg = CLI.IsVarArg;
5207 bool IsThisReturn =
false;
5211 bool IsSibCall =
false;
5220 return In.VT.isScalableVector();
5223 if (CalleeInSVE || CalleeOutSVE)
5229 IsTailCall = isEligibleForTailCallOptimization(
5230 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
5231 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
5233 "site marked musttail");
5237 if (!TailCallOpt && IsTailCall)
5252 unsigned NumArgs = Outs.
size();
5254 for (
unsigned i = 0; i != NumArgs; ++i) {
5255 MVT ArgVT = Outs[i].VT;
5258 "currently not supported");
5264 assert(!Res &&
"Call operand has unhandled type");
5274 unsigned NumArgs = Outs.
size();
5275 for (
unsigned i = 0; i != NumArgs; ++i) {
5276 MVT ValVT = Outs[i].VT;
5279 CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
5291 assert(!Res &&
"Call operand has unhandled type");
5312 if (IsTailCall && !IsSibCall) {
5317 NumBytes =
alignTo(NumBytes, 16);
5322 FPDiff = NumReusableBytes - NumBytes;
5329 assert(FPDiff % 16 == 0 &&
"unaligned stack on tail call");
5345 if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
5347 for (
const auto &
F : Forwards) {
5354 unsigned ExtraArgLocs = 0;
5355 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
5373 if (Outs[i].ArgVT ==
MVT::i1) {
5397 "Only scalable vectors can be passed indirectly");
5400 uint64_t PartSize = StoreSize;
5401 unsigned NumParts = 1;
5402 if (Outs[i].Flags.isInConsecutiveRegs()) {
5403 assert(!Outs[i].Flags.isInConsecutiveRegsLast());
5404 while (!Outs[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
5406 StoreSize *= NumParts;
5424 Chain = DAG.
getStore(Chain,
DL, OutVals[i], Ptr, MPI);
5435 BytesIncrement, Flags);
5446 if (i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
5449 "unexpected calling convention register assignment");
5451 "unexpected use of 'returned'");
5452 IsThisReturn =
true;
5461 [=](
const std::pair<unsigned, SDValue> &Elt) {
5492 OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
5494 OpSize = (OpSize + 7) / 8;
5496 !Flags.isInConsecutiveRegs()) {
5498 BEAlign = 8 - OpSize;
5501 int32_t
Offset = LocMemOffset + BEAlign;
5516 Chain = addTokenForArgument(Chain, DAG, MF.
getFrameInfo(), FI);
5525 if (Outs[i].Flags.isByVal()) {
5529 Chain,
DL, DstAddr,
Arg, SizeNode,
5530 Outs[i].Flags.getNonZeroByValAlign(),
5549 if (!MemOpChains.
empty())
5555 for (
auto &RegToPass : RegsToPass) {
5557 RegToPass.second, InFlag);
5565 auto GV =
G->getGlobal();
5578 const char *Sym = S->getSymbol();
5582 const char *Sym = S->getSymbol();
5591 if (IsTailCall && !IsSibCall) {
5597 std::vector<SDValue>
Ops;
5598 Ops.push_back(Chain);
5610 for (
auto &RegToPass : RegsToPass)
5612 RegToPass.second.getValueType()));
5619 Mask =
TRI->getThisReturnPreservedMask(MF, CallConv);
5621 IsThisReturn =
false;
5622 Mask =
TRI->getCallPreservedMask(MF, CallConv);
5625 Mask =
TRI->getCallPreservedMask(MF, CallConv);
5628 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
5630 if (
TRI->isAnyArgRegReserved(MF))
5631 TRI->emitReservedArgRegCallError(MF);
5633 assert(Mask &&
"Missing call preserved mask for calling convention");
5637 Ops.push_back(InFlag);
5654 if (CLI.CB && CLI.CB->hasRetAttr(
"rv_marker")) {
5655 assert(!IsTailCall &&
"tail calls cannot be marked with rv_marker");
5665 uint64_t CalleePopBytes =
5666 DoesCalleeRestoreStack(CallConv, TailCallOpt) ?
alignTo(NumBytes, 16) : 0;
5676 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins,
DL, DAG,
5677 InVals, IsThisReturn,
5678 IsThisReturn ? OutVals[0] :
SDValue());
5681bool AArch64TargetLowering::CanLowerReturn(
5686 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
5691AArch64TargetLowering::LowerReturn(
SDValue Chain, CallingConv::ID CallConv,
5709 for (
unsigned i = 0, realRVLocIdx = 0; i != RVLocs.
size();
5710 ++i, ++realRVLocIdx) {
5719 if (Outs[i].ArgVT ==
MVT::i1) {
5744 llvm::find_if(RetVals, [=](
const std::pair<unsigned, SDValue> &Elt) {
5755 for (
auto &RetVal : RetVals) {
5756 Chain = DAG.
getCopyToReg(Chain,
DL, RetVal.first, RetVal.second, Flag);
5759 DAG.
getRegister(RetVal.first, RetVal.second.getValueType()));
5770 unsigned RetValReg = AArch64::X0;
5783 if (AArch64::GPR64RegClass.
contains(*
I))
5785 else if (AArch64::FPR64RegClass.
contains(*
I))
5796 RetOps.push_back(Flag);
5807 unsigned Flag)
const {
5809 N->getOffset(), Flag);
5814 unsigned Flag)
const {
5820 unsigned Flag)
const {
5822 N->getOffset(), Flag);
5827 unsigned Flag)
const {
5832template <
class NodeTy>
5834 unsigned Flags)
const {
5845template <
class NodeTy>
5847 unsigned Flags)
const {
5861template <
class NodeTy>
5863 unsigned Flags)
const {
5875template <
class NodeTy>
5877 unsigned Flags)
const {
5881 SDValue Sym = getTargetNode(
N, Ty, DAG, Flags);
5893 "unexpected offset in global node");
5898 return getGOT(GN, DAG, OpFlags);
5903 Result = getAddrLarge(GN, DAG, OpFlags);
5905 Result = getAddrTiny(GN, DAG, OpFlags);
5907 Result = getAddr(GN, DAG, OpFlags);
5946AArch64TargetLowering::LowerDarwinGlobalTLSAddress(
SDValue Op,
5949 "This function expects a Darwin target");
5964 PtrMemVT,
DL, Chain, DescAddr,
6104SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(
SDValue SymAddr,
6120AArch64TargetLowering::LowerELFGlobalTLSAddress(
SDValue Op,
6136 "in local exec TLS model");
6152 return LowerELFTLSLocalExec(GV, ThreadBase,
DL, DAG);
6175 TPOff = LowerELFTLSDescCallSeq(SymAddr,
DL, DAG);
6199 TPOff = LowerELFTLSDescCallSeq(SymAddr,
DL, DAG);
6207AArch64TargetLowering::LowerWindowsGlobalTLSAddress(
SDValue Op,
6272 return LowerDarwinGlobalTLSAddress(Op, DAG);
6274 return LowerELFGlobalTLSAddress(Op, DAG);
6276 return LowerWindowsGlobalTLSAddress(Op, DAG);
6309 bool ProduceNonFlagSettingCondBr =
6320 if (!RHS.getNode()) {
6340 OFCC = getInvertedCondCode(OFCC);
6347 if (LHS.getValueType().isInteger()) {
6348 assert((LHS.getValueType() == RHS.getValueType()) &&
6354 if (RHSC && RHSC->
getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
6364 uint64_t
Mask = LHS.getConstantOperandVal(1);
6380 uint64_t
Mask = LHS.getConstantOperandVal(1);
6391 uint64_t SignBitPos;
6398 LHS.getOpcode() !=
ISD::AND && ProduceNonFlagSettingCondBr) {
6402 uint64_t SignBitPos;
6436 EVT VT =
Op.getValueType();
6445 else if (SrcVT.
bitsGT(VT))
6452 auto setVecVal = [&] (
int Idx) {
6466 EltMask = 0x80000000ULL;
6467 setVecVal(AArch64::ssub);
6476 setVecVal(AArch64::dsub);
6479 EltMask = 0x8000ULL;
6480 setVecVal(AArch64::hsub);
6510 Attribute::NoImplicitFloat))
6526 EVT VT =
Op.getValueType();
6557 "Unexpected type for custom ctpop lowering");
6564 unsigned EltSize = 8;
6579 EVT VT =
Op.getValueType();
6581 useSVEForFixedLengthVectorVT(VT,
true));
6590 if (
Op.getValueType().isVector())
6591 return LowerVSETCC(Op, DAG);
6593 bool IsStrict =
Op->isStrictFPOpcode();
6595 unsigned OpNo = IsStrict ? 1 : 0;
6598 Chain =
Op.getOperand(0);
6605 EVT VT =
Op.getValueType();
6616 if (!RHS.getNode()) {
6617 assert(LHS.getValueType() ==
Op.getValueType() &&
6618 "Unexpected setcc expansion!");
6623 if (LHS.getValueType().isInteger()) {
6687 if (!RHS.getNode()) {
6700 if (LHS.getValueType().isInteger()) {
6701 assert((LHS.getValueType() == RHS.getValueType()) &&
6735 }
else if (CTVal && CFVal) {
6743 if (TrueVal == ~FalseVal) {
6745 }
else if (FalseVal > std::numeric_limits<int64_t>::min() &&
6746 TrueVal == -FalseVal) {
6757 if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
6760 if (TrueVal32 > FalseVal32) {
6765 }
else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
6768 if (TrueVal > FalseVal) {
6800 else if (CFVal && CFVal == RHSVal && AArch64CC ==
AArch64CC::NE)
6803 assert (CTVal && CFVal &&
"Expected constant operands for CSNEG.");
6817 return DAG.
getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
6823 assert(LHS.getValueType() == RHS.getValueType());
6836 if (RHSVal && RHSVal->
isZero()) {
6844 CFVal && CFVal->
isZero() &&
6873 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal,
DL, DAG);
6883 EVT Ty =
Op.getValueType();
6884 if (Ty.isScalableVector()) {
6919 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal,
DL, DAG);
6930 return getAddrLarge(JT, DAG);
6932 return getAddrTiny(JT, DAG);
6934 return getAddr(JT, DAG);
6963 return getGOT(CP, DAG);
6965 return getAddrLarge(CP, DAG);
6967 return getAddrTiny(CP, DAG);
6969 return getAddr(CP, DAG);
6978 return getAddrLarge(BA, DAG);
6980 return getAddrTiny(BA, DAG);
6982 return getAddr(BA, DAG);
7098 return LowerWin64_VASTART(Op, DAG);
7100 return LowerDarwin_VASTART(Op, DAG);
7102 return LowerAAPCS_VASTART(Op, DAG);
7111 unsigned VaListSize =
7120 Align(PtrSize),
false,
false,
false,
7126 "automatic va_arg instruction only works on Darwin");
7129 EVT VT =
Op.getValueType();
7144 "currently not supported");
7161 ArgSize = std::max(ArgSize, MinSlotSize);
7162 bool NeedFPTrunc =
false;
7198 EVT VT =
Op.getValueType();
7224#define GET_REGISTER_MATCHER
7225#include "AArch64GenAsmMatcher.inc"
7232 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
7234 unsigned DwarfRegNum =
MRI->getDwarfRegNum(Reg,
false);
7248 EVT VT =
Op.getValueType();
7264 EVT VT =
Op.getValueType();
7269 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
7277 unsigned Reg = MF.
addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
7301 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
7302 EVT VT =
Op.getValueType();
7323 HiBitsForLo, CCVal, Cmp);
7335 SDValue LoForBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
7337 LoForNormalShift, CCVal, Cmp);
7341 SDValue HiForNormalShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ShAmt);
7344 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
7348 HiForNormalShift, CCVal, Cmp);
7358 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
7359 EVT VT =
Op.getValueType();
7378 LoBitsForHi, CCVal, Cmp);
7392 HiForNormalShift, CCVal, Cmp);
7399 LoForNormalShift, CCVal, Cmp);
7413 bool OptForSize)
const {
7414 bool IsLegal =
false;
7440 unsigned Limit = (OptForSize ? 1 : (Subtarget->
hasFuseLiterals() ? 5 : 2));
7441 IsLegal = Insn.
size() <= Limit;
7445 <<
" imm value: "; Imm.
dump(););
7457 if (ST->hasNEON() &&
7469 return DAG.
getNode(Opcode,
SDLoc(Operand), VT, Operand);
7479 EVT VT =
Op.getValueType();
7486AArch64TargetLowering::getSqrtResultForDenormInput(
SDValue Op,
7495 bool Reciprocal)
const {
7508 for (
int i = ExtraSteps; i > 0; --i) {
7526 int &ExtraSteps)
const {
7538 for (
int i = ExtraSteps; i > 0; --i) {
7578const char *AArch64TargetLowering::LowerXConstraint(
EVT ConstraintVT)
const {
7608 if (Constraint ==
"Upa")
7610 if (Constraint ==
"Upl")
7618AArch64TargetLowering::getConstraintType(
StringRef Constraint)
const {
7619 if (Constraint.
size() == 1) {
7620 switch (Constraint[0]) {
7654AArch64TargetLowering::getSingleConstraintMatchWeight(
7655 AsmOperandInfo &
info,
const char *constraint)
const {
7657 Value *CallOperandVal =
info.CallOperandVal;
7660 if (!CallOperandVal)
7664 switch (*constraint) {
7671 if (type->isFloatingPointTy() || type->isVectorTy())
7685std::pair<unsigned, const TargetRegisterClass *>
7686AArch64TargetLowering::getRegForInlineAsmConstraint(
7688 if (Constraint.
size() == 1) {
7689 switch (Constraint[0]) {
7692 return std::make_pair(0U,
nullptr);
7694 return std::make_pair(0U, &AArch64::GPR64commonRegClass);
7695 return std::make_pair(0U, &AArch64::GPR32commonRegClass);
7701 return std::make_pair(0U, &AArch64::ZPRRegClass);
7702 return std::make_pair(0U,
nullptr);
7706 return std::make_pair(0U, &AArch64::FPR16RegClass);
7708 return std::make_pair(0U, &AArch64::FPR32RegClass);
7710 return std::make_pair(0U, &AArch64::FPR64RegClass);
7712 return std::make_pair(0U, &AArch64::FPR128RegClass);
7721 return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
7723 return std::make_pair(0U, &AArch64::FPR128_loRegClass);
7729 return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
7736 return std::make_pair(0U,
nullptr);
7738 return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
7739 :
std::make_pair(0U, &AArch64::PPRRegClass);
7742 if (
StringRef(
"{cc}").equals_lower(Constraint))
7743 return std::make_pair(
unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
7747 std::pair<unsigned, const TargetRegisterClass *> Res;
7753 if ((
Size == 4 ||
Size == 5) && Constraint[0] ==
'{' &&
7754 tolower(Constraint[1]) ==
'v' && Constraint[
Size - 1] ==
'}') {
7757 if (!
Failed && RegNo >= 0 && RegNo <= 31) {
7762 Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
7763 Res.second = &AArch64::FPR64RegClass;
7765 Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
7766 Res.second = &AArch64::FPR128RegClass;
7772 if (Res.second && !Subtarget->
hasFPARMv8() &&
7773 !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
7774 !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
7775 return std::make_pair(0U,
nullptr);
7782void AArch64TargetLowering::LowerAsmOperandForConstraint(
7783 SDValue Op, std::string &Constraint, std::vector<SDValue> &
Ops,
7788 if (Constraint.length() != 1)
7791 char ConstraintLetter = Constraint[0];
7792 switch (ConstraintLetter) {
7838 uint64_t CVal =
C->getZExtValue();
7839 switch (ConstraintLetter) {
7851 uint64_t NVal = -
C->getSExtValue();
7853 CVal =
C->getSExtValue();
7884 if ((CVal & 0xFFFF) == CVal)
7886 if ((CVal & 0xFFFF0000ULL) == CVal)
7889 if ((NCVal & 0xFFFFULL) == NCVal)
7891 if ((NCVal & 0xFFFF0000ULL) == NCVal)
7898 if ((CVal & 0xFFFFULL) == CVal)
7900 if ((CVal & 0xFFFF0000ULL) == CVal)
7902 if ((CVal & 0xFFFF00000000ULL) == CVal)
7904 if ((CVal & 0xFFFF000000000000ULL) == CVal)
7906 uint64_t NCVal = ~CVal;
7907 if ((NCVal & 0xFFFFULL) == NCVal)
7909 if ((NCVal & 0xFFFF0000ULL) == NCVal)
7911 if ((NCVal & 0xFFFF00000000ULL) == NCVal)
7913 if ((NCVal & 0xFFFF000000000000ULL) == NCVal)
7927 Ops.push_back(Result);
7975 LLVM_DEBUG(
dbgs() <<
"AArch64TargetLowering::ReconstructShuffle\n");
7977 EVT VT = Op.getValueType();
7979 "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
7982 struct ShuffleSourceInfo {
7997 ShuffleSourceInfo(
SDValue Vec)
7998 : Vec(Vec), MinElt(std::numeric_limits<unsigned>::max()), MaxElt(0),
7999 ShuffleVec(Vec), WindowBase(0), WindowScale(1) {}
8007 for (
unsigned i = 0; i < NumElts; ++i) {
8014 dbgs() <<
"Reshuffle failed: "
8015 "a shuffle can only come from building a vector from "
8016 "various elements of other vectors, provided their "
8017 "indices are constant\n");
8023 auto Source =
find(Sources, SourceVec);
8024 if (Source == Sources.
end())
8025 Source = Sources.
insert(Sources.
end(), ShuffleSourceInfo(SourceVec));
8029 Source->MinElt = std::min(Source->MinElt, EltNo);
8030 Source->MaxElt = std::max(Source->MaxElt, EltNo);
8033 if (Sources.
size() > 2) {
8035 dbgs() <<
"Reshuffle failed: currently only do something sane when at "
8036 "most two source vectors are involved\n");
8043 for (
auto &Source : Sources) {
8044 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8045 if (SrcEltTy.
bitsLT(SmallestEltTy)) {
8046 SmallestEltTy = SrcEltTy;
8049 unsigned ResMultiplier =
8058 for (
auto &Src : Sources) {
8059 EVT SrcVT = Src.ShuffleVec.getValueType();
8062 if (SrcVTSize == VTSize)
8071 if (SrcVTSize < VTSize) {
8072 assert(2 * SrcVTSize == VTSize);
8077 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8081 if (SrcVTSize != 2 * VTSize) {
8083 dbgs() <<
"Reshuffle failed: result vector too small to extract\n");
8087 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8089 dbgs() <<
"Reshuffle failed: span too large for a VEXT to cope\n");
8093 if (Src.MinElt >= NumSrcElts) {
8098 Src.WindowBase = -NumSrcElts;
8099 }
else if (Src.MaxElt < NumSrcElts) {
8116 dbgs() <<
"Reshuffle failed: don't know how to lower AArch64ISD::EXT "
8117 "for SVE vectors.");
8124 Src.WindowBase = -Src.MinElt;
8131 for (
auto &Src : Sources) {
8133 if (SrcEltTy == SmallestEltTy)
8139 Src.WindowBase *= Src.WindowScale;
8145 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8151 SDValue Entry = Op.getOperand(i);
8152 if (Entry.isUndef())
8155 auto Src =
find(Sources, Entry.getOperand(0));
8164 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8168 int *LaneMask = &Mask[i * ResMultiplier];
8170 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8171 ExtractBase += NumElts * (Src - Sources.
begin());
8172 for (
int j = 0; j < LanesDefined; ++j)
8173 LaneMask[j] = ExtractBase + j;
8178 LLVM_DEBUG(
dbgs() <<
"Reshuffle failed: illegal shuffle mask\n");
8183 for (
unsigned i = 0; i < Sources.
size(); ++i)
8191 dbgs() <<
"Reshuffle, creating node: "; V.dump(););
8210 unsigned ExpectedElt = Imm;
8211 for (
unsigned i = 1; i < NumElts; ++i) {
8215 if (ExpectedElt == NumElts)
8220 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
8232 unsigned &DupLaneOp) {
8234 "Only possible block sizes for wide DUP are: 16, 32, 64");
8253 for (
size_t BlockIndex = 0; BlockIndex < NumBlocks; BlockIndex++)
8254 for (
size_t I = 0;
I < NumEltsPerBlock;
I++) {
8255 int Elt = M[BlockIndex * NumEltsPerBlock +
I];
8259 if ((
unsigned)Elt >= SingleVecNumElements)
8261 if (BlockElts[
I] < 0)
8263 else if (BlockElts[
I] != Elt)
8272 auto FirstRealEltIter =
find_if(BlockElts, [](
int Elt) {
return Elt >= 0; });
8273 assert(FirstRealEltIter != BlockElts.
end() &&
8274 "Shuffle with all-undefs must have been caught by previous cases, "
8276 if (FirstRealEltIter == BlockElts.
end()) {
8282 size_t FirstRealIndex = FirstRealEltIter - BlockElts.
begin();
8284 if ((
unsigned)*FirstRealEltIter < FirstRealIndex)
8287 size_t Elt0 = *FirstRealEltIter - FirstRealIndex;
8290 if (Elt0 % NumEltsPerBlock != 0)
8294 for (
size_t I = 0;
I < NumEltsPerBlock;
I++)
8295 if (BlockElts[
I] >= 0 && (
unsigned)BlockElts[
I] != Elt0 +
I)
8298 DupLaneOp = Elt0 / NumEltsPerBlock;
8307 const int *FirstRealElt =
find_if(M, [](
int Elt) {
return Elt >= 0; });
8312 APInt ExpectedElt =
APInt(MaskBits, *FirstRealElt + 1);
8315 const int *FirstWrongElt = std::find_if(FirstRealElt + 1, M.end(),
8316 [&](
int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
8317 if (FirstWrongElt != M.end())
8347 "Only possible block sizes for REV are: 16, 32, 64");
8354 unsigned BlockElts = M[0] + 1;
8362 for (
unsigned i = 0; i < NumElts; ++i) {
8365 if ((
unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
8374 if (NumElts % 2 != 0)
8376 WhichResult = (M[0] == 0 ? 0 : 1);
8377 unsigned Idx = WhichResult * NumElts / 2;
8378 for (
unsigned i = 0; i != NumElts; i += 2) {
8379 if ((M[i] >= 0 && (
unsigned)M[i] != Idx) ||
8380 (M[i + 1] >= 0 && (
unsigned)M[i + 1] != Idx + NumElts))
8390 WhichResult = (M[0] == 0 ? 0 : 1);
8391 for (
unsigned i = 0; i != NumElts; ++i) {
8394 if ((
unsigned)M[i] != 2 * i + WhichResult)
8403 if (NumElts % 2 != 0)
8405 WhichResult = (M[0] == 0 ? 0 : 1);
8406 for (
unsigned i = 0; i < NumElts; i += 2) {
8407 if ((M[i] >= 0 && (
unsigned)M[i] != i + WhichResult) ||
8408 (M[i + 1] >= 0 && (
unsigned)M[i + 1] != i + NumElts + WhichResult))
8419 if (NumElts % 2 != 0)
8421 WhichResult = (M[0] == 0 ? 0 : 1);
8422 unsigned Idx = WhichResult * NumElts / 2;
8423 for (
unsigned i = 0; i != NumElts; i += 2) {
8424 if ((M[i] >= 0 && (
unsigned)M[i] != Idx) ||
8425 (M[i + 1] >= 0 && (
unsigned)M[i + 1] != Idx))
8438 WhichResult = (M[0] == 0 ? 0 : 1);
8439 for (
unsigned j = 0; j != 2; ++j) {
8440 unsigned Idx = WhichResult;
8441 for (
unsigned i = 0; i != Half; ++i) {
8442 int MIdx = M[i + j * Half];
8443 if (MIdx >= 0 && (
unsigned)MIdx != Idx)
8457 if (NumElts % 2 != 0)
8459 WhichResult = (M[0] == 0 ? 0 : 1);
8460 for (
unsigned i = 0; i < NumElts; i += 2) {
8461 if ((M[i] >= 0 && (
unsigned)M[i] != i + WhichResult) ||
8462 (M[i + 1] >= 0 && (
unsigned)M[i + 1] != i + WhichResult))
8469 bool &DstIsLeft,
int &Anomaly) {
8470 if (M.size() !=
static_cast<size_t>(NumInputElements))
8473 int NumLHSMatch = 0, NumRHSMatch = 0;
8474 int LastLHSMismatch = -1, LastRHSMismatch = -1;
8476 for (
int i = 0; i < NumInputElements; ++i) {
8486 LastLHSMismatch = i;
8488 if (M[i] == i + NumInputElements)
8491 LastRHSMismatch = i;
8494 if (NumLHSMatch == NumInputElements - 1) {
8496 Anomaly = LastLHSMismatch;
8498 }
else if (NumRHSMatch == NumInputElements - 1) {
8500 Anomaly = LastRHSMismatch;
8513 for (
int I = 0,
E = NumElts / 2;
I !=
E;
I++) {
8518 int Offset = NumElts / 2;
8519 for (
int I = NumElts / 2,
E = NumElts;
I !=
E;
I++) {
8520 if (Mask[
I] !=
I + SplitLHS *
Offset)
8529 EVT VT = Op.getValueType();
8530 SDValue V0 = Op.getOperand(0);
8531 SDValue V1 = Op.getOperand(1);
8560 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8561 unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1);
8562 unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1);
8583 if (LHSID == (1 * 9 + 2) * 9 + 3)
8585 assert(LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 &&
"Illegal OP_COPY!");
8630 return DAG.
getNode(Opcode, dl, VT, OpLHS, Lane);
8663 SDValue V1 = Op.getOperand(0);
8664 SDValue V2 = Op.getOperand(1);
8667 EVT EltVT = Op.getValueType().getVectorElementType();
8671 for (
int Val : ShuffleMask) {
8672 for (
unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
8673 unsigned Offset = Byte + Val * BytesPerElt;
8679 unsigned IndexLen = 8;
8680 if (Op.getValueSizeInBits() == 128) {
8689 if (V2.getNode()->isUndef()) {
8698 if (IndexLen == 8) {
8738 auto getScaledOffsetDup = [](
SDValue BitCast,
int &LaneC,
MVT &CastVT) {
8749 unsigned ExtIdxInBits = ExtIdx * SrcEltBitWidth;
8751 if (ExtIdxInBits % CastedEltBitWidth != 0)
8755 LaneC += ExtIdxInBits / CastedEltBitWidth;
8762 unsigned SrcVecNumElts =
8769 if (getScaledOffsetDup(V, Lane, CastVT)) {
8770 V = DAG.
getBitcast(CastVT, V.getOperand(0).getOperand(0));
8774 Lane += V.getConstantOperandVal(1);
8775 V = V.getOperand(0);
8792 EVT VT =
Op.getValueType();
8826 for (
unsigned LaneSize : {64U, 32U, 16U}) {
8838 V1 =
constructDup(V1, Lane, dl, NewVecTy, Opcode, DAG);
8851 bool ReverseEXT =
false;
8853 if (
isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
8865 unsigned WhichResult;
8866 if (
isZIPMask(ShuffleMask, VT, WhichResult)) {
8870 if (
isUZPMask(ShuffleMask, VT, WhichResult)) {
8874 if (
isTRNMask(ShuffleMask, VT, WhichResult)) {
8898 if (
isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
8903 int SrcLane = ShuffleMask[Anomaly];
8904 if (SrcLane >= NumInputElements) {
8925 unsigned PFIndexes[4];
8926 for (
unsigned i = 0; i != 4; ++i) {
8927 if (ShuffleMask[i] < 0)
8930 PFIndexes[i] = ShuffleMask[i];
8934 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
8935 PFIndexes[2] * 9 + PFIndexes[3];
8937 unsigned Cost = (PFEntry >> 30);
8949 EVT VT =
Op.getValueType();
8953 if (useSVEForFixedLengthVectorVT(VT))
8954 return LowerToScalableOp(Op, DAG);
8963 if (ConstVal->isOne())
8964 return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
9002 EVT VT =
Op.getValueType();
9016 if (CIdx && (CIdx->getZExtValue() <= 3)) {
9050 APInt SplatBits, SplatUndef;
9051 unsigned SplatBitSize;
9053 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
9056 for (
unsigned i = 0; i < NumSplats; ++i) {
9057 CnstBits <<= SplatBitSize;
9058 UndefBits <<= SplatBitSize;
9060 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.
getSizeInBits());
9071 const APInt &Bits) {
9072 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9073 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9074 EVT VT = Op.getValueType();
9093 const SDValue *LHS =
nullptr) {
9094 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9095 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9096 EVT VT = Op.getValueType();
9098 bool isAdvSIMDModImm =
false;
9118 if (isAdvSIMDModImm) {
9123 Mov = DAG.
getNode(NewOp, dl, MovTy, *LHS,
9127 Mov = DAG.
getNode(NewOp, dl, MovTy,
9141 const SDValue *LHS =
nullptr) {
9142 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9143 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9144 EVT VT = Op.getValueType();
9146 bool isAdvSIMDModImm =
false;
9158 if (isAdvSIMDModImm) {
9163 Mov = DAG.
getNode(NewOp, dl, MovTy, *LHS,
9167 Mov = DAG.
getNode(NewOp, dl, MovTy,
9181 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9182 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9183 EVT VT = Op.getValueType();
9185 bool isAdvSIMDModImm =
false;
9197 if (isAdvSIMDModImm) {
9211 const APInt &Bits) {
9212 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9213 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9214 EVT VT = Op.getValueType();
9232 const APInt &Bits) {
9233 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9234 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9235 EVT VT = Op.getValueType();
9238 bool isAdvSIMDModImm =
false;
9250 if (isAdvSIMDModImm) {
9265 uint64_t &ConstVal) {
9274 for (
unsigned i = 1; i < NumElts; ++i)
9282 unsigned Opcode =
N->getOpcode();
9302 EVT VT =
N->getValueType(0);
9313 unsigned FirstOpc = FirstOp.
getOpcode();
9315 unsigned SecondOpc = SecondOp.
getOpcode();
9350 assert(C1nodeImm && C1nodeShift);
9359 if (C2 > ElemSizeInBits)
9362 APInt C1AsAPInt(ElemSizeInBits, C1);
9365 if (C1AsAPInt != RequiredC1)
9385 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
9386 return LowerToScalableOp(Op, DAG);
9392 EVT VT =
Op.getValueType();
9399 LHS =
Op.getOperand(1);
9417 UndefBits, &LHS)) ||
9433 EVT VT = Op.getValueType();
9440 for (
SDValue Lane : Op->ops()) {
9447 CstLane->getZExtValue());
9449 }
else if (Lane.getNode()->isUndef()) {
9453 "Unexpected BUILD_VECTOR operand type");
9455 Ops.push_back(Lane);
9461 EVT VT = Op.getValueType();
9482 DefBits = UndefBits;
9491 DefBits = ~UndefBits;
9503 EVT VT =
Op.getValueType();
9517 Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
9518 if (Val.isNullValue() || Val.isAllOnesValue())
9540 bool isOnlyLowElement =
true;
9541 bool usesOnlyOneValue =
true;
9542 bool usesOnlyOneConstantValue =
true;
9544 bool AllLanesExtractElt =
true;
9545 unsigned NumConstantLanes = 0;
9546 unsigned NumDifferentLanes = 0;
9547 unsigned NumUndefLanes = 0;
9550 for (
unsigned i = 0; i < NumElts; ++i) {
9553 AllLanesExtractElt =
false;
9559 isOnlyLowElement =
false;
9567 else if (ConstantValue != V)
9568 usesOnlyOneConstantValue =
false;
9571 if (!
Value.getNode())
9573 else if (V !=
Value) {
9574 usesOnlyOneValue =
false;
9575 ++NumDifferentLanes;
9579 if (!
Value.getNode()) {
9581 dbgs() <<
"LowerBUILD_VECTOR: value undefined, creating undef node\n");
9589 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: only low element used, creating 1 "
9590 "SCALAR_TO_VECTOR node\n");
9594 if (AllLanesExtractElt) {
9600 for (
unsigned i = 0; i < NumElts; ++i) {
9602 const SDNode *
N = V.getNode();
9628 if (Val - 1 == 2 * i) {
9657 if (usesOnlyOneValue) {
9660 Value.getValueType() != VT) {
9662 dbgs() <<
"LowerBUILD_VECTOR: use DUP for non-constant splats\n");
9670 if (
Value.getValueSizeInBits() == 64) {
9672 dbgs() <<
"LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
9685 EltTy ==
MVT::f64) &&
"Unsupported floating-point vector type");
9687 dbgs() <<
"LowerBUILD_VECTOR: float constant splats, creating int "
9688 "BITCASTS, and try again\n");
9690 for (
unsigned i = 0; i < NumElts; ++i)
9694 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: trying to lower new vector: ";
9696 Val = LowerBUILD_VECTOR(Val, DAG);
9706 bool PreferDUPAndInsert =
9708 NumDifferentLanes < ((NumElts - NumUndefLanes) / 2) &&
9709 NumDifferentLanes >= NumConstantLanes;
9715 if (!PreferDUPAndInsert && NumConstantLanes > 0 && usesOnlyOneConstantValue) {
9726 for (
unsigned i = 0; i < NumElts; ++i) {
9740 dbgs() <<
"LowerBUILD_VECTOR: all elements are constant, use default "
9751 if (PreferDUPAndInsert) {
9754 for (
unsigned I = 0;
I < NumElts; ++
I)
9758 for (
unsigned I = 0;
I < NumElts; ++
I)
9775 dbgs() <<
"LowerBUILD_VECTOR: alternatives failed, creating sequence "
9776 "of INSERT_VECTOR_ELT\n");
9793 LLVM_DEBUG(
dbgs() <<
"Creating node for op0, it is not undefined:\n");
9798 <<
"Creating nodes for the other vector elements:\n";);
9799 for (; i < NumElts; ++i) {
9810 dbgs() <<
"LowerBUILD_VECTOR: use default expansion, failed to find "
9811 "better alternative\n");
9817 assert(
Op.getValueType().isScalableVector() &&
9819 "Expected legal scalable vector type!");
9821 if (
isTypeLegal(
Op.getOperand(0).getValueType()) &&
Op.getNumOperands() == 2)
9832 EVT VT =
Op.getOperand(0).getValueType();
9856 Op.getOperand(1),
Op.getOperand(2));
9862AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(
SDValue Op,
9867 EVT VT =
Op.getOperand(0).getValueType();
9901 assert(
Op.getValueType().isFixedLengthVector() &&
9902 "Only cases that extract a fixed length vector are supported!");
9904 EVT InVT =
Op.getOperand(0).getValueType();
9906 unsigned Size =
Op.getValueSizeInBits();
9931 assert(
Op.getValueType().isScalableVector() &&
9932 "Only expect to lower inserts into scalable vectors!");
9934 EVT InVT =
Op.getOperand(1).getValueType();
9939 EVT VT =
Op.getValueType();
9974 EVT VT =
Op.getValueType();
9976 if (useSVEForFixedLengthVectorVT(VT,
true))
9977 return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
9985 return LowerToPredicatedOp(Op, DAG, PredOpcode);
10004 SDValue ResultLo = DAG.
getNode(
Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
10005 SDValue ResultHi = DAG.
getNode(
Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
10011 if (useSVEForFixedLengthVectorVT(VT))
10016 unsigned PFIndexes[4];
10017 for (
unsigned i = 0; i != 4; ++i) {
10021 PFIndexes[i] = M[i];
10025 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10026 PFIndexes[2] * 9 + PFIndexes[3];
10028 unsigned Cost = (PFEntry >> 30);
10036 unsigned DummyUnsigned;
10040 isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
10057 Op = Op.getOperand(0);
10059 APInt SplatBits, SplatUndef;
10060 unsigned SplatBitSize;
10062 if (!BVN || !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10063 HasAnyUndefs, ElementBits) ||
10064 SplatBitSize > ElementBits)
10075 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10079 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
10086 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10090 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
10095 EVT VT =
Op.getValueType();
10100 EVT OpVT =
Op.getOperand(0).getValueType();
10110 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10111 return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
10116SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(
SDValue Op,
10118 EVT VT =
Op.getValueType();
10122 if (!
Op.getOperand(1).getValueType().isVector())
10126 switch (
Op.getOpcode()) {
10134 if (
isVShiftLImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize)
10140 Op.getOperand(0),
Op.getOperand(1));
10146 return LowerToPredicatedOp(Op, DAG, Opc);
10150 if (
isVShiftRImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize) {
10153 return DAG.
getNode(Opc,
DL, VT,
Op.getOperand(0),
10160 unsigned Opc = (
Op.getOpcode() ==
ISD::SRA) ? Intrinsic::aarch64_neon_sshl
10161 : Intrinsic::aarch64_neon_ushl;
10168 return NegShiftLeft;
10177 EVT SrcVT = LHS.getValueType();
10179 "function only supposed to emit natural comparisons");
10185 bool IsZero = IsCnst && (CnstBits == 0);
10197 return DAG.
getNOT(dl, Fcmeq, VT);
10236 return DAG.
getNOT(dl, Cmeq, VT);
10271 if (
Op.getValueType().isScalableVector()) {
10272 if (
Op.getOperand(0).getValueType().isFloatingPoint())
10277 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10278 return LowerFixedLengthVectorSetccToSVE(Op, DAG);
10286 if (LHS.getValueType().getVectorElementType().isInteger()) {
10287 assert(LHS.getValueType() == RHS.getValueType());
10294 const bool FullFP16 =
10299 if (!FullFP16 && LHS.getValueType().getVectorElementType() ==
MVT::f16) {
10300 if (LHS.getValueType().getVectorNumElements() == 4) {
10310 assert((!FullFP16 && LHS.getValueType().getVectorElementType() !=
MVT::f16) ||
10311 LHS.getValueType().getVectorElementType() !=
MVT::f128);
10322 if (!
Cmp.getNode())
10363 useSVEForFixedLengthVectorVT(SrcVT, OverrideNEON)) {
10366 return LowerPredReductionToSVE(Op, DAG);
10368 switch (
Op.getOpcode()) {
10398 switch (
Op.getOpcode()) {
10434 MVT VT =
Op.getSimpleValueType();
10439 Op.getOperand(0),
Op.getOperand(1), RHS,
10451 MVT VT =
Op.getSimpleValueType();
10456 Op.getOperand(0),
Op.getOperand(1), RHS,
10460SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
10489AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(
SDValue Op,
10492 "Only Windows alloca probing supported");
10500 EVT VT =
Node->getValueType(0);
10503 "no-stack-arg-probe")) {
10517 Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain,
Size, DAG);
10536 EVT VT =
Op.getValueType();
10546template <
unsigned NumVecs>
10556 for (
unsigned I = 0;
I < NumVecs; ++
I)
10565 Info.align.reset();
10576 unsigned Intrinsic)
const {
10577 auto &
DL =
I.getModule()->getDataLayout();
10578 switch (Intrinsic) {
10579 case Intrinsic::aarch64_sve_st2:
10581 case Intrinsic::aarch64_sve_st3:
10583 case Intrinsic::aarch64_sve_st4:
10585 case Intrinsic::aarch64_neon_ld2:
10586 case Intrinsic::aarch64_neon_ld3:
10587 case Intrinsic::aarch64_neon_ld4:
10588 case Intrinsic::aarch64_neon_ld1x2:
10589 case Intrinsic::aarch64_neon_ld1x3:
10590 case Intrinsic::aarch64_neon_ld1x4:
10591 case Intrinsic::aarch64_neon_ld2lane:
10592 case Intrinsic::aarch64_neon_ld3lane:
10593 case Intrinsic::aarch64_neon_ld4lane:
10594 case Intrinsic::aarch64_neon_ld2r:
10595 case Intrinsic::aarch64_neon_ld3r:
10596 case Intrinsic::aarch64_neon_ld4r: {
10599 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
10601 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
10603 Info.align.reset();
10608 case Intrinsic::aarch64_neon_st2:
10609 case Intrinsic::aarch64_neon_st3:
10610 case Intrinsic::aarch64_neon_st4:
10611 case Intrinsic::aarch64_neon_st1x2:
10612 case Intrinsic::aarch64_neon_st1x3:
10613 case Intrinsic::aarch64_neon_st1x4:
10614 case Intrinsic::aarch64_neon_st2lane:
10615 case Intrinsic::aarch64_neon_st3lane:
10616 case Intrinsic::aarch64_neon_st4lane: {
10619 unsigned NumElts = 0;
10620 for (
unsigned ArgI = 0, ArgE =
I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
10621 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
10624 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
10627 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
10629 Info.align.reset();
10634 case Intrinsic::aarch64_ldaxr:
10635 case Intrinsic::aarch64_ldxr: {
10639 Info.ptrVal =
I.getArgOperand(0);
10645 case Intrinsic::aarch64_stlxr:
10646 case Intrinsic::aarch64_stxr: {
10650 Info.ptrVal =
I.getArgOperand(1);
10656 case Intrinsic::aarch64_ldaxp:
10657 case Intrinsic::aarch64_ldxp:
10660 Info.ptrVal =
I.getArgOperand(0);
10662 Info.align =
Align(16);
10665 case Intrinsic::aarch64_stlxp:
10666 case Intrinsic::aarch64_stxp:
10669 Info.ptrVal =
I.getArgOperand(2);
10671 Info.align =
Align(16);
10674 case Intrinsic::aarch64_sve_ldnt1: {
10678 Info.ptrVal =
I.getArgOperand(1);
10682 if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
10686 case Intrinsic::aarch64_sve_stnt1: {
10689 Info.memVT =
MVT::getVT(
I.getOperand(0)->getType());
10690 Info.ptrVal =
I.getArgOperand(2);
10694 if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
10729 if (ShiftAmount ==
Log2_32(LoadBytes))
10742 return NumBits1 > NumBits2;
10749 return NumBits1 > NumBits2;
10756 if (
I->getOpcode() != Instruction::FMul)
10759 if (!
I->hasOneUse())
10765 !(
User->getOpcode() == Instruction::FSub ||
10766 User->getOpcode() == Instruction::FAdd))
10787 return NumBits1 == 32 && NumBits2 == 64;
10794 return NumBits1 == 32 && NumBits2 == 64;
10812bool AArch64TargetLowering::isExtFreeImpl(
const Instruction *Ext)
const {
10817 if (Ext->getType()->isVectorTy())
10820 for (
const Use &U : Ext->uses()) {
10829 case Instruction::Shl:
10833 case Instruction::GetElementPtr: {
10835 auto &
DL = Ext->getModule()->getDataLayout();
10836 std::advance(GTI, U.getOperandNo()-1);
10842 uint64_t ShiftAmt =
10846 if (ShiftAmt == 0 || ShiftAmt > 4)
10850 case Instruction::Trunc:
10853 if (Instr->
getType() == Ext->getOperand(0)->getType())
10869 auto areTypesHalfed = [](
Value *FullV,
Value *HalfV) {
10870 auto *FullTy = FullV->
getType();
10871 auto *HalfTy = HalfV->getType();
10873 2 * HalfTy->getPrimitiveSizeInBits().getFixedSize();
10876 auto extractHalf = [](
Value *FullV,
Value *HalfV) {
10879 return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
10883 Value *S1Op1, *S2Op1;
10890 if (!areTypesHalfed(S1Op1, Op1) || !areTypesHalfed(S2Op1, Op2) ||
10891 !extractHalf(S1Op1, Op1) || !extractHalf(S2Op1, Op2))
10901 M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2)))
10911 return Ext->getType()->getScalarSizeInBits() ==
10912 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
10926 Value *VectorOperand =
nullptr;
10945 if (!
I->getType()->isVectorTy())
10949 switch (II->getIntrinsicID()) {
10950 case Intrinsic::aarch64_neon_umull:
10953 Ops.push_back(&II->getOperandUse(0));
10954 Ops.push_back(&II->getOperandUse(1));
10957 case Intrinsic::aarch64_neon_pmull64:
10959 II->getArgOperand(1)))
10961 Ops.push_back(&II->getArgOperandUse(0));
10962 Ops.push_back(&II->getArgOperandUse(1));
10970 switch (
I->getOpcode()) {
10971 case Instruction::Sub:
10972 case Instruction::Add: {
10981 Ops.push_back(&Ext1->getOperandUse(0));
10982 Ops.push_back(&Ext2->getOperandUse(0));
10985 Ops.push_back(&
I->getOperandUse(0));
10986 Ops.push_back(&
I->getOperandUse(1));
10990 case Instruction::Mul: {
10991 bool IsProfitable =
false;
10992 for (
auto &Op :
I->operands()) {
10994 if (
any_of(
Ops, [&](
Use *U) {
return U->get() == Op; }))
11013 if (!ElementConstant || ElementConstant->
getZExtValue() != 0)
11016 unsigned Opcode = OperandInstr->
getOpcode();
11017 if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
11021 Ops.push_back(&Op);
11022 IsProfitable =
true;
11025 return IsProfitable;
11034 Align &RequiredAligment)
const {
11039 RequiredAligment =
Align(1);
11041 return NumBits == 32 || NumBits == 64;
11049 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
11063 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
11071 if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
11076 return VecSize == 64 || VecSize % 128 == 0;
11094 "Invalid interleave factor");
11095 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
11097 "Unmatched number of shufflevectors and indices");
11115 Type *EltTy = FVTy->getElementType();
11125 if (NumLoads > 1) {
11129 FVTy->getNumElements() / NumLoads);
11134 BaseAddr = Builder.CreateBitCast(
11140 Type *Tys[2] = {FVTy, PtrTy};
11141 static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
11142 Intrinsic::aarch64_neon_ld3,
11143 Intrinsic::aarch64_neon_ld4};
11152 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
11157 BaseAddr = Builder.CreateConstGEP1_32(FVTy->getElementType(), BaseAddr,
11158 FVTy->getNumElements() * Factor);
11160 CallInst *LdN = Builder.CreateCall(
11161 LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy),
"ldN");
11164 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
11166 unsigned Index = Indices[i];
11168 Value *SubVec = Builder.CreateExtractValue(LdN, Index);
11172 SubVec = Builder.CreateIntToPtr(
11174 FVTy->getNumElements()));
11175 SubVecs[SVI].push_back(SubVec);
11184 auto &SubVec = SubVecs[SVI];
11187 SVI->replaceAllUsesWith(WideVec);
11221 unsigned Factor)
const {
11223 "Invalid interleave factor");
11226 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
11228 unsigned LaneLen = VecTy->getNumElements() / Factor;
11229 Type *EltTy = VecTy->getElementType();
11249 Type *IntTy =
DL.getIntPtrType(EltTy);
11250 unsigned NumOpElts =
11255 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
11256 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
11262 Value *BaseAddr =
SI->getPointerOperand();
11264 if (NumStores > 1) {
11267 LaneLen /= NumStores;
11273 BaseAddr = Builder.CreateBitCast(
11275 SubVecTy->getElementType()->getPointerTo(
SI->getPointerAddressSpace()));
11280 Type *PtrTy = SubVecTy->getPointerTo(
SI->getPointerAddressSpace());
11281 Type *Tys[2] = {SubVecTy, PtrTy};
11282 static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2,
11283 Intrinsic::aarch64_neon_st3,
11284 Intrinsic::aarch64_neon_st4};
11288 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
11293 for (
unsigned i = 0; i < Factor; i++) {
11294 unsigned IdxI = StoreCount * LaneLen * Factor + i;
11295 if (Mask[IdxI] >= 0) {
11296 Ops.push_back(Builder.CreateShuffleVector(
11299 unsigned StartMask = 0;
11300 for (
unsigned j = 1; j < LaneLen; j++) {
11301 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
11302 if (Mask[IdxJ * Factor + IdxI] >= 0) {
11303 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
11312 Ops.push_back(Builder.CreateShuffleVector(
11319 if (StoreCount > 0)
11320 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
11321 BaseAddr, LaneLen * Factor);
11323 Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
11324 Builder.CreateCall(StNFunc,
Ops);
11349SDValue AArch64TargetLowering::LowerSVEStructLoad(
unsigned Intrinsic,
11355 unsigned N, Opcode;
11356 static std::map<unsigned, std::pair<unsigned, unsigned>> IntrinsicMap = {
11361 std::tie(
N, Opcode) = IntrinsicMap[
Intrinsic];
11363 "invalid tuple vector type!");
11376 for (
unsigned I = 0;
I <
N; ++
I)
11383 bool CanImplicitFloat =
11385 bool CanUseNEON = Subtarget->
hasNEON() && CanImplicitFloat;
11386 bool CanUseFP = Subtarget->
hasFPARMv8() && CanImplicitFloat;
11390 bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11391 auto AlignmentIsAcceptable = [&](
EVT VT,
Align AlignCheck) {
11392 if (Op.isAligned(AlignCheck))
11400 if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11403 if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(
MVT::f128,
Align(16)))
11405 if (Op.size() >= 8 && AlignmentIsAcceptable(
MVT::i64,
Align(8)))
11407 if (Op.size() >= 4 && AlignmentIsAcceptable(
MVT::i32,
Align(4)))
11414 bool CanImplicitFloat =
11416 bool CanUseNEON = Subtarget->
hasNEON() && CanImplicitFloat;
11417 bool CanUseFP = Subtarget->
hasFPARMv8() && CanImplicitFloat;
11421 bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11422 auto AlignmentIsAcceptable = [&](
EVT VT,
Align AlignCheck) {
11423 if (Op.isAligned(AlignCheck))
11431 if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11434 if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(
MVT::f128,
Align(16)))
11436 if (Op.size() >= 8 && AlignmentIsAcceptable(
MVT::i64,
Align(8)))
11438 if (Op.size() >= 4 && AlignmentIsAcceptable(
MVT::i32,
Align(4)))
11445 if (Immed == std::numeric_limits<int64_t>::min()) {
11447 <<
": avoid UB for INT64_MIN\n");
11451 Immed = std::abs(Immed);
11452 bool IsLegal = ((Immed >> 12) == 0 ||
11453 ((Immed & 0xfff) == 0 && Immed >> 24 == 0));
11455 <<
" legal add imm: " << (IsLegal ?
"yes" :
"no") <<
"\n");
11491 uint64_t NumBytes = 0;
11492 if (Ty->isSized()) {
11493 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
11494 NumBytes = NumBits / 8;
11507 unsigned shift =
Log2_64(NumBytes);
11508 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
11517 return AM.
Scale == 1 || (AM.
Scale > 0 && (uint64_t)AM.
Scale == NumBytes);
11527 unsigned AS)
const {
11562 switch (Ty->getScalarType()->getTypeID()) {
11576 static const MCPhysReg ScratchRegs[] = {
11577 AArch64::X16, AArch64::X17, AArch64::LR, 0
11579 return ScratchRegs;
11585 N =
N->getOperand(0).getNode();
11586 EVT VT =
N->getValueType(0);
11591 uint64_t TruncMask =
N->getConstantOperandVal(1);
11593 N->getOperand(0).getOpcode() ==
ISD::SRL &&
11602 assert(Ty->isIntegerTy());
11604 unsigned BitSize = Ty->getPrimitiveSizeInBits();
11612 if ((int64_t)Val < 0)
11615 Val &= (1LL << 32) - 1;
11618 unsigned Shift = (63 - LZ) / 16;
11624 unsigned Index)
const {
11637 EVT VT =
N->getValueType(0);
11652 if (!ShiftAmt || ShiftAmt->getZExtValue() != ShiftEltTy.
getSizeInBits() - 1)
11663 if (!ST->hasDotProd() ||
N->getValueType(0) !=
MVT::i32)
11681 ? Intrinsic::aarch64_neon_udot
11682 : Intrinsic::aarch64_neon_sdot;
11714 if (VectorT1 != VectorT2 || (
Size != 64 &&
Size != 128))
11724 unsigned ABDOpcode =
11741AArch64TargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
11749 EVT VT =
N->getValueType(0);
11751 !(Divisor.
isPowerOf2() || (-Divisor).isPowerOf2()))
11787 case Intrinsic::aarch64_sve_cntb:
11788 case Intrinsic::aarch64_sve_cnth:
11789 case Intrinsic::aarch64_sve_cntw:
11790 case Intrinsic::aarch64_sve_cntd:
11819 return TypeNode->
getVT();
11829 if (Mask == UCHAR_MAX)
11831 else if (Mask == USHRT_MAX)
11833 else if (Mask == UINT_MAX)
11871 unsigned ExtendOpcode = Extend.
getOpcode();
11912 DAG.
getUNDEF(PreExtendVT), ShuffleMask);
11915 DL, TargetType, VectorShuffleNode);
11924 if (!
Mul->getValueType(0).isVector())
11956 const APInt &ConstValue =
C->getAPIntValue();
11963 if (ConstValue.
sge(1) && ConstValue.
sle(16))
11979 if (TrailingZeroes) {
11987 if (
N->hasOneUse() && (
N->use_begin()->getOpcode() ==
ISD::ADD ||
11988 N->use_begin()->getOpcode() ==
ISD::SUB))
11993 APInt ShiftedConstValue = ConstValue.
ashr(TrailingZeroes);
11995 unsigned ShiftAmt, AddSubOpc;
11997 bool ShiftValUseIsN0 =
true;
11999 bool NegateResult =
false;
12005 APInt SCVMinus1 = ShiftedConstValue - 1;
12006 APInt CVPlus1 = ConstValue + 1;
12018 APInt CVNegPlus1 = -ConstValue + 1;
12019 APInt CVNegMinus1 = -ConstValue - 1;
12023 ShiftValUseIsN0 =
false;
12025 ShiftAmt = CVNegMinus1.
logBase2();
12027 NegateResult =
true;
12033 EVT VT =
N->getValueType(0);
12037 SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
12038 SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
12040 assert(!(NegateResult && TrailingZeroes) &&
12041 "NegateResult and TrailingZeroes cannot both be true for now.");
12046 if (TrailingZeroes)
12065 EVT VT =
N->getValueType(0);
12067 N->getOperand(0)->getOperand(0)->getOpcode() !=
ISD::SETCC ||
12068 VT.
getSizeInBits() !=
N->getOperand(0)->getValueType(0).getSizeInBits())
12078 if (!BV->isConstant())
12083 EVT IntVT = BV->getValueType(0);
12090 N->getOperand(0)->getOperand(0), MaskConst);
12105 EVT VT =
N->getValueType(0);
12110 if (VT.
getSizeInBits() !=
N->getOperand(0).getValueSizeInBits())
12145 if (!
N->getValueType(0).isSimple())
12149 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12153 SDValue ConstVec = Op->getOperand(1);
12157 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
12159 if (FloatBits != 32 && FloatBits != 64)
12163 uint32_t IntBits = IntTy.getSizeInBits();
12164 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12168 if (IntBits > FloatBits)
12173 int32_t Bits = IntBits == 64 ? 64 : 32;
12175 if (
C == -1 ||
C == 0 ||
C > Bits)
12179 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12180 switch (NumLanes) {
12195 "Illegal vector type after legalization");
12199 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
12200 : Intrinsic::aarch64_neon_vcvtfp2fxu;
12206 if (IntBits < FloatBits)
12221 unsigned Opc = Op->getOpcode();
12222 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12223 !Op.getOperand(0).getValueType().isSimple() ||
12231 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
12232 int32_t IntBits = IntTy.getSizeInBits();
12233 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12238 if (FloatBits != 32 && FloatBits != 64)
12242 if (IntBits > FloatBits)
12248 if (
C == -1 ||
C == 0 ||
C > FloatBits)
12252 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12253 switch (NumLanes) {
12268 SDValue ConvInput = Op.getOperand(0);
12270 if (IntBits < FloatBits)
12274 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
12275 : Intrinsic::aarch64_neon_vcvtfxu2fp;
12295 ShiftAmount =
N->getConstantOperandVal(1);
12309 EVT VT =
N->getValueType(0);
12318 bool LHSFromHi =
false;
12319 if (!
findEXTRHalf(
N->getOperand(0), LHS, ShiftLHS, LHSFromHi))
12324 bool RHSFromHi =
false;
12325 if (!
findEXTRHalf(
N->getOperand(1), RHS, ShiftRHS, RHSFromHi))
12330 if (LHSFromHi == RHSFromHi)
12347 EVT VT =
N->getValueType(0);
12365 uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
12366 for (
int i = 1; i >= 0; --i)
12367 for (
int j = 1; j >= 0; --j) {
12370 if (!BVN0 || !BVN1)
12373 bool FoundMatch =
true;
12377 if (!CN0 || !CN1 ||
12379 FoundMatch =
false;
12396 EVT VT =
N->getValueType(0);
12414 uint64_t MaskForTy = 0ull;
12417 MaskForTy = 0xffull;
12420 MaskForTy = 0xffffull;
12423 MaskForTy = 0xffffffffull;
12432 return Op0->getAPIntValue().getLimitedValue() == MaskForTy;
12456 uint64_t ExtVal =
C->getZExtValue();
12461 if ((ExtVal == 0xFF && EltTy ==
MVT::i8) ||
12462 (ExtVal == 0xFFFF && EltTy ==
MVT::i16) ||
12463 (ExtVal == 0xFFFFFFFF && EltTy ==
MVT::i32))
12530 EVT VT =
N->getValueType(0);
12556 DefBits = ~DefBits;
12563 UndefBits = ~UndefBits;
12565 UndefBits, &LHS)) ||
12577 EVT VT =
N->getValueType(0);
12590 uint64_t ShiftAmt =
C->getZExtValue();
12591 if (VT ==
MVT::i32 && ShiftAmt == 16 &&
12594 if (VT ==
MVT::i64 && ShiftAmt == 32 &&
12616 EVT VT =
N->getValueType(0);
12629 if (ShiftAmount != 1)
12632 SDValue ExtendOpA, ExtendOpB;
12634 unsigned ShiftOp0Opc = ShiftOp0.
getOpcode();
12647 APInt CAsAPInt(ElemSizeInBits,
C);
12653 }
else if (ShiftOp0Opc ==
ISD::ADD) {
12659 unsigned ExtendOpAOpc = ExtendOpA.
getOpcode();
12660 unsigned ExtendOpBOpc = ExtendOpB.
getOpcode();
12661 if (!(ExtendOpAOpc == ExtendOpBOpc &&
12676 bool IsRHADD = ShiftOp0Opc ==
ISD::SUB;
12677 unsigned HADDOpc = IsSignExtend
12700 EVT VT =
N->getValueType(0);
12701 const bool FullFP16 =
12726 if (Shuffle && Shuffle->
getMaskElt(0) == 1 &&
12743 EVT VT =
N->getValueType(0);
12769 for (
size_t i = 0; i < Mask.size(); ++i)
12795 if (
N->getNumOperands() == 2 && N0Opc == N1Opc &&
12815 if (N00Source == N10Source && N01Source == N11Source &&
12824 if (N00Index == N01Index && N10Index == N11Index && N00Index == 0 &&
12826 return DAG.
getNode(N0Opc, dl, VT, N00Source, N01Source);
12852 MVT RHSTy = RHS.getValueType().getSimpleVT();
12858 dbgs() <<
"aarch64-lower: concat_vectors bitcast simplification\n");
12892 EVT ResTy =
N->getValueType(0);
12900 "unexpected vector size on extract_vector_elt!");
12931 switch (
N.getOpcode()) {
12951 MVT NarrowTy =
N.getSimpleValueType();
12961 DAG.
getNode(
N->getOpcode(), dl, NewVT,
N->ops()),
12967 N =
N.getOperand(0);
12971 N.getOperand(0).getValueType().getVectorNumElements() / 2;
13036 if (!TValue || !FValue)
13040 if (!TValue->
isOne()) {
13054 isSetCC(Op->getOperand(0), Info));
13064 assert(Op && Op->getOpcode() ==
ISD::ADD &&
"Unexpected operation!");
13065 SDValue LHS = Op->getOperand(0);
13066 SDValue RHS = Op->getOperand(1);
13097 EVT VT = Op->getValueType(0);
13104 EVT VT =
N->getValueType(0);
13117 if (!LHSN1 || LHSN1 != RHSN1 || !RHSN1->isNullValue())
13156 MVT VT =
N->getSimpleValueType(0);
13168 LHS.getOpcode() != RHS.getOpcode())
13171 unsigned ExtType = LHS.getOpcode();
13177 if (!RHS.getNode())
13183 if (!LHS.getNode())
13217 assert(LHS.getValueType().is64BitVector() &&
13218 RHS.getValueType().is64BitVector() &&
13219 "unexpected shape for long operation");
13226 if (!RHS.getNode())
13230 if (!LHS.getNode())
13235 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N),
N->getValueType(0), LHS, RHS);
13238 N->getOperand(0), LHS, RHS);
13245 int64_t ShiftAmount;
13247 APInt SplatValue, SplatUndef;
13248 unsigned SplatBitSize;
13251 HasAnyUndefs, ElemBits) ||
13252 SplatBitSize != ElemBits)
13257 ShiftAmount = CVN->getSExtValue();
13266 case Intrinsic::aarch64_neon_sqshl:
13268 IsRightShift =
false;
13270 case Intrinsic::aarch64_neon_uqshl:
13272 IsRightShift =
false;
13274 case Intrinsic::aarch64_neon_srshl:
13276 IsRightShift =
true;
13278 case Intrinsic::aarch64_neon_urshl:
13280 IsRightShift =
true;
13282 case Intrinsic::aarch64_neon_sqshlu:
13284 IsRightShift =
false;
13286 case Intrinsic::aarch64_neon_sshl:
13287 case Intrinsic::aarch64_neon_ushl:
13292 IsRightShift =
false;
13296 if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(
int)ElemBits) {
13298 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
13300 }
else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
13302 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
13322 N->getOperand(0),
N->getOperand(1), AndN.
getOperand(0));
13330 N->getOperand(1).getSimpleValueType(),
13352 SDValue Scalar =
N->getOperand(3);
13353 EVT ScalarTy = Scalar.getValueType();
13361 Pred, Scalar, Passthru);
13367 EVT VT =
N->getValueType(0);
13400 EVT VT =
N->getValueType(0);
13401 EVT CmpVT =
N->getOperand(2).getValueType();
13412 case Intrinsic::aarch64_sve_cmpeq_wide:
13413 case Intrinsic::aarch64_sve_cmpne_wide:
13414 case Intrinsic::aarch64_sve_cmpge_wide:
13415 case Intrinsic::aarch64_sve_cmpgt_wide:
13416 case Intrinsic::aarch64_sve_cmplt_wide:
13417 case Intrinsic::aarch64_sve_cmple_wide: {
13419 int64_t ImmVal = CN->getSExtValue();
13420 if (ImmVal >= -16 && ImmVal <= 15)
13428 case Intrinsic::aarch64_sve_cmphs_wide:
13429 case Intrinsic::aarch64_sve_cmphi_wide:
13430 case Intrinsic::aarch64_sve_cmplo_wide:
13431 case Intrinsic::aarch64_sve_cmpls_wide: {
13433 uint64_t ImmVal = CN->getZExtValue();
13459 assert(Op.getValueType().isScalableVector() &&
13461 "Expected legal scalable vector type!");
13527 DAG.
getUNDEF(ReduceVT), InitVal, Zero);
13529 SDValue Reduce = DAG.
getNode(Opc,
DL, ReduceVT, Pred, InitVal, VecToReduce);
13543 assert(
N->getNumOperands() == 4 &&
"Expected 3 operand intrinsic!");
13564 case Intrinsic::aarch64_neon_vcvtfxs2fp:
13565 case Intrinsic::aarch64_neon_vcvtfxu2fp:
13567 case Intrinsic::aarch64_neon_saddv:
13569 case Intrinsic::aarch64_neon_uaddv:
13571 case Intrinsic::aarch64_neon_sminv:
13573 case Intrinsic::aarch64_neon_uminv:
13575 case Intrinsic::aarch64_neon_smaxv:
13577 case Intrinsic::aarch64_neon_umaxv:
13579 case Intrinsic::aarch64_neon_fmax:
13581 N->getOperand(1),
N->getOperand(2));
13582 case Intrinsic::aarch64_neon_fmin:
13584 N->getOperand(1),
N->getOperand(2));
13585 case Intrinsic::aarch64_neon_fmaxnm:
13587 N->getOperand(1),
N->getOperand(2));
13588 case Intrinsic::aarch64_neon_fminnm:
13590 N->getOperand(1),
N->getOperand(2));
13591 case Intrinsic::aarch64_neon_smull:
13592 case Intrinsic::aarch64_neon_umull:
13593 case Intrinsic::aarch64_neon_pmull:
13594 case Intrinsic::aarch64_neon_sqdmull:
13596 case Intrinsic::aarch64_neon_sqshl:
13597 case Intrinsic::aarch64_neon_uqshl:
13598 case Intrinsic::aarch64_neon_sqshlu:
13599 case Intrinsic::aarch64_neon_srshl:
13600 case Intrinsic::aarch64_neon_urshl:
13601 case Intrinsic::aarch64_neon_sshl:
13602 case Intrinsic::aarch64_neon_ushl:
13604 case Intrinsic::aarch64_crc32b:
13605 case Intrinsic::aarch64_crc32cb:
13607 case Intrinsic::aarch64_crc32h:
13608 case Intrinsic::aarch64_crc32ch:
13610 case Intrinsic::aarch64_sve_saddv:
13612 if (
N->getOperand(2)->getValueType(0).getVectorElementType() ==
MVT::i64)
13616 case Intrinsic::aarch64_sve_uaddv:
13618 case Intrinsic::aarch64_sve_smaxv:
13620 case Intrinsic::aarch64_sve_umaxv:
13622 case Intrinsic::aarch64_sve_sminv:
13624 case Intrinsic::aarch64_sve_uminv:
13626 case Intrinsic::aarch64_sve_orv:
13628 case Intrinsic::aarch64_sve_eorv:
13630 case Intrinsic::aarch64_sve_andv:
13632 case Intrinsic::aarch64_sve_index:
13634 case Intrinsic::aarch64_sve_dup:
13636 case Intrinsic::aarch64_sve_dup_x:
13639 case Intrinsic::aarch64_sve_ext:
13641 case Intrinsic::aarch64_sve_smin:
13643 case Intrinsic::aarch64_sve_umin:
13645 case Intrinsic::aarch64_sve_smax:
13647 case Intrinsic::aarch64_sve_umax:
13649 case Intrinsic::aarch64_sve_lsl:
13651 case Intrinsic::aarch64_sve_lsr:
13653 case Intrinsic::aarch64_sve_asr:
13655 case Intrinsic::aarch64_sve_cmphs:
13656 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13658 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13661 case Intrinsic::aarch64_sve_cmphi:
13662 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13664 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13667 case Intrinsic::aarch64_sve_cmpge:
13668 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13670 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13673 case Intrinsic::aarch64_sve_cmpgt:
13674 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13676 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13679 case Intrinsic::aarch64_sve_cmpeq:
13680 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13682 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13685 case Intrinsic::aarch64_sve_cmpne:
13686 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13688 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13691 case Intrinsic::aarch64_sve_fadda:
13693 case Intrinsic::aarch64_sve_faddv:
13695 case Intrinsic::aarch64_sve_fmaxnmv:
13697 case Intrinsic::aarch64_sve_fmaxv:
13699 case Intrinsic::aarch64_sve_fminnmv:
13701 case Intrinsic::aarch64_sve_fminv:
13703 case Intrinsic::aarch64_sve_sel:
13705 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
13706 case Intrinsic::aarch64_sve_cmpeq_wide:
13708 case Intrinsic::aarch64_sve_cmpne_wide:
13710 case Intrinsic::aarch64_sve_cmpge_wide:
13712 case Intrinsic::aarch64_sve_cmpgt_wide:
13714 case Intrinsic::aarch64_sve_cmplt_wide:
13716 case Intrinsic::aarch64_sve_cmple_wide:
13718 case Intrinsic::aarch64_sve_cmphs_wide:
13720 case Intrinsic::aarch64_sve_cmphi_wide:
13722 case Intrinsic::aarch64_sve_cmplo_wide:
13724 case Intrinsic::aarch64_sve_cmpls_wide:
13726 case Intrinsic::aarch64_sve_ptest_any:
13729 case Intrinsic::aarch64_sve_ptest_first:
13732 case Intrinsic::aarch64_sve_ptest_last:
13789 EVT ResVT =
N->getValueType(0);
13809 Src = DAG.
getNode(
N->getOpcode(),
DL, SrcVT, Src);
13823 Lo = DAG.
getNode(
N->getOpcode(),
DL, LoVT, Lo);
13824 Hi = DAG.
getNode(
N->getOpcode(),
DL, HiVT, Hi);
13832 SDValue SplatVal,
unsigned NumVecElts) {
13843 uint64_t BaseOffset = 0;
13851 if (BasePtr->getOpcode() ==
ISD::ADD &&
13854 BasePtr = BasePtr->getOperand(0);
13857 unsigned Offset = EltOffset;
13858 while (--NumVecElts) {
13874 assert(ContentTy.
isSimple() &&
"No SVE containers for extended types");
13903 EVT VT =
N->getValueType(0);
13908 EVT ContainerVT = VT;
13921 if (ContainerVT.
isInteger() && (VT != ContainerVT))
13929 EVT VT =
N->getValueType(0);
13930 EVT PtrTy =
N->getOperand(3).getValueType();
13943 MINode->getOperand(3), DAG.
getUNDEF(PtrTy),
13945 MINode->getMemoryVT(), MINode->getMemOperand(),
13956template <
unsigned Opcode>
13960 "Unsupported opcode.");
13962 EVT VT =
N->getValueType(0);
13971 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(2),
N->getOperand(3)};
13984 EVT DataVT =
Data.getValueType();
13996 if (
Data.getValueType().isFloatingPoint())
14015 EVT DataVT =
Data.getValueType();
14016 EVT PtrTy =
N->getOperand(4).getValueType();
14028 MINode->getMemoryVT(), MINode->getMemOperand(),
14058 if (!(((NumVecElts == 2 || NumVecElts == 3) &&
14060 ((NumVecElts == 2 || NumVecElts == 3 || NumVecElts == 4) &&
14086 for (
int I = 0;
I < NumVecElts; ++
I) {
14098 ZeroReg = AArch64::WZR;
14101 ZeroReg = AArch64::XZR;
14125 if (NumVecElts != 4 && NumVecElts != 2)
14136 std::bitset<4> IndexNotInserted((1 << NumVecElts) - 1);
14138 for (
unsigned I = 0;
I < NumVecElts; ++
I) {
14154 if (IndexVal >= NumVecElts)
14156 IndexNotInserted.reset(IndexVal);
14161 if (IndexNotInserted.any())
14185 return ReplacedZeroSplat;
14216 return ReplacedSplat;
14242 EVT ResVT =
N->getValueType(0);
14272 EVT VT =
N->getValueType(0);
14277 unsigned LoadIdx = IsLaneOp ? 1 : 0;
14278 SDNode *LD =
N->getOperand(LoadIdx).getNode();
14302 if (UI.getUse().getResNo() == 1)
14312 Addr.getNode()->use_end(); UI != UE; ++UI) {
14315 || UI.getUse().getResNo() !=
Addr.getResNo())
14321 uint32_t IncVal = CInc->getZExtValue();
14323 if (IncVal != NumBytes)
14344 Ops.push_back(Lane);
14347 Ops.push_back(Inc);
14372static bool performTBISimplification(
SDValue Addr,
14395 performTBISimplification(
N->getOperand(2), DCI, DAG))
14405 assert(MGS &&
"Can only combine gather load or scatter store nodes");
14415 EVT IdxVT = Index.getValueType();
14458 unsigned AddrOpIdx =
N->getNumOperands() - 1;
14463 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
14466 UI.getUse().getResNo() !=
Addr.getResNo())
14481 bool IsStore =
false;
14482 bool IsLaneOp =
false;
14483 bool IsDupOp =
false;
14484 unsigned NewOpc = 0;
14485 unsigned NumVecs = 0;
14490 NumVecs = 2;
break;
14492 NumVecs = 3;
break;
14494 NumVecs = 4;
break;
14496 NumVecs = 2; IsStore =
true;
break;
14498 NumVecs = 3; IsStore =
true;
break;
14500 NumVecs = 4; IsStore =
true;
break;
14502 NumVecs = 2;
break;
14504 NumVecs = 3;
break;
14506 NumVecs = 4;
break;
14508 NumVecs = 2; IsStore =
true;
break;
14510 NumVecs = 3; IsStore =
true;
break;
14512 NumVecs = 4; IsStore =
true;
break;
14514 NumVecs = 2; IsDupOp =
true;
break;
14516 NumVecs = 3; IsDupOp =
true;
break;
14518 NumVecs = 4; IsDupOp =
true;
break;
14520 NumVecs = 2; IsLaneOp =
true;
break;
14522 NumVecs = 3; IsLaneOp =
true;
break;
14524 NumVecs = 4; IsLaneOp =
true;
break;
14526 NumVecs = 2; IsStore =
true; IsLaneOp =
true;
break;
14528 NumVecs = 3; IsStore =
true; IsLaneOp =
true;
break;
14530 NumVecs = 4; IsStore =
true; IsLaneOp =
true;
break;
14535 VecTy =
N->getOperand(2).getValueType();
14537 VecTy =
N->getValueType(0);
14542 uint32_t IncVal = CInc->getZExtValue();
14544 if (IsLaneOp || IsDupOp)
14546 if (IncVal != NumBytes)
14553 if (IsLaneOp || IsStore)
14554 for (
unsigned i = 2; i < AddrOpIdx; ++i)
14555 Ops.push_back(
N->getOperand(i));
14557 Ops.push_back(Inc);
14561 unsigned NumResultVecs = (IsStore ? 0 : NumVecs);
14563 for (n = 0; n < NumResultVecs; ++n)
14575 std::vector<SDValue> NewResults;
14576 for (
unsigned i = 0; i < NumResultVecs; ++i) {
14579 NewResults.push_back(
SDValue(UpdN.
getNode(), NumResultVecs + 1));
14593 switch(V.getNode()->getOpcode()) {
14626 1LL << (width - 1);
14694static bool isEquivalentMaskless(
unsigned CC,
unsigned width,
14696 int CompConstant) {
14700 int MaxUInt = (1 << width);
14708 AddConstant -= (1 << (width-1));
14713 if ((AddConstant == 0) ||
14714 (CompConstant == MaxUInt - 1 && AddConstant < 0) ||
14715 (AddConstant >= 0 && CompConstant < 0) ||
14716 (AddConstant <= 0 && CompConstant <= 0 && CompConstant < AddConstant))
14721 if ((AddConstant == 0) ||
14722 (AddConstant >= 0 && CompConstant <= 0) ||
14723 (AddConstant <= 0 && CompConstant <= 0 && CompConstant <= AddConstant))
14728 if ((AddConstant >= 0 && CompConstant < 0) ||
14729 (AddConstant <= 0 && CompConstant >= -1 &&
14730 CompConstant < AddConstant + MaxUInt))
14735 if ((AddConstant == 0) ||
14736 (AddConstant > 0 && CompConstant <= 0) ||
14737 (AddConstant < 0 && CompConstant <= AddConstant))
14742 if ((AddConstant >= 0 && CompConstant <= 0) ||
14743 (AddConstant <= 0 && CompConstant >= 0 &&
14744 CompConstant <= AddConstant + MaxUInt))
14749 if ((AddConstant > 0 && CompConstant < 0) ||
14750 (AddConstant < 0 && CompConstant >= 0 &&
14751 CompConstant < AddConstant + MaxUInt) ||
14752 (AddConstant >= 0 && CompConstant >= 0 &&
14753 CompConstant >= AddConstant) ||
14754 (AddConstant <= 0 && CompConstant < 0 && CompConstant < AddConstant))
14773 unsigned CmpIndex) {
14776 unsigned CondOpcode = SubsNode->
getOpcode();
14785 unsigned MaskBits = 0;
14791 uint32_t CNV = CN->getZExtValue();
14794 else if (CNV == 65535)
14821 if (!checkValueWidth(SubsInputValue, MaskBits, ExtType) ||
14822 !checkValueWidth(AddInputValue2, MaskBits, ExtType) ||
14823 !checkValueWidth(AddInputValue1, MaskBits, ExtType) )
14826 if(!isEquivalentMaskless(CC, MaskBits, ExtType,
14854 if (
SDValue NV = performCONDCombine(
N, DCI, DAG, 2, 3))
14866 unsigned CmpOpc =
Cmp.getOpcode();
14872 if (!
Cmp->hasNUsesOfValue(0, 0) || !
Cmp->hasNUsesOfValue(1, 1))
14878 assert(LHS.getValueType() == RHS.getValueType() &&
14879 "Expected the value type to be the same for both operands!");
14910static SDValue getTestBitOperand(
SDValue Op,
unsigned &Bit,
bool &Invert,
14913 if (!
Op->hasOneUse())
14923 Bit < Op->getValueType(0).getSizeInBits()) {
14924 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14929 Bit < Op->getOperand(0).getValueSizeInBits()) {
14930 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14933 if (
Op->getNumOperands() != 2)
14940 switch (
Op->getOpcode()) {
14946 if ((
C->getZExtValue() >> Bit) & 1)
14947 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14952 if (
C->getZExtValue() <= Bit &&
14953 (Bit -
C->getZExtValue()) <
Op->getValueType(0).getSizeInBits()) {
14954 Bit =
Bit -
C->getZExtValue();
14955 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14961 Bit =
Bit +
C->getZExtValue();
14962 if (Bit >=
Op->getValueType(0).getSizeInBits())
14963 Bit =
Op->getValueType(0).getSizeInBits() - 1;
14964 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14968 if ((Bit +
C->getZExtValue()) <
Op->getValueType(0).getSizeInBits()) {
14969 Bit =
Bit +
C->getZExtValue();
14970 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14976 if ((
C->getZExtValue() >> Bit) & 1)
14978 return getTestBitOperand(
Op->getOperand(0), Bit, Invert, DAG);
14987 bool Invert =
false;
14989 SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
14991 if (TestSrc == NewTestSrc)
14994 unsigned NewOpc =
N->getOpcode();
15022 EVT ResVT =
N->getValueType(0);
15047 EVT ResVT =
N->getValueType(0);
15056 "Scalar-SETCC feeding SELECT has unexpected result type!");
15069 if (!ResVT.
isVector() || NumMaskElts == 0)
15100 return DAG.
getSelect(
DL, ResVT, Mask,
N->getOperand(1),
N->getOperand(2));
15105 if (
N->getValueType(0) ==
N->getOperand(0).getValueType())
15122 uint64_t MinOffset = -1ull;
15131 MinOffset = std::min(MinOffset,
C->getZExtValue());
15149 if (
Offset >= (1 << 21))
15154 if (!
T->isSized() ||
15169 "This method is only for scalable vectors of offsets");
15185inline static bool isValidImmForSVEVecImmAddrMode(
unsigned OffsetInBytes,
15186 unsigned ScalarSizeInBytes) {
15188 if (OffsetInBytes % ScalarSizeInBytes)
15192 if (OffsetInBytes / ScalarSizeInBytes > 31)
15206 unsigned ScalarSizeInBytes) {
15208 return OffsetConst && isValidImmForSVEVecImmAddrMode(
15214 bool OnlyPackedOffsets =
true) {
15218 "Scatter stores are only possible for SVE vectors");
15263 if (!isValidImmForSVEVecImmAddrMode(
Offset,
15281 if (!OnlyPackedOffsets &&
15318 bool OnlyPackedOffsets =
true) {
15319 const EVT RetVT =
N->getValueType(0);
15321 "Gather loads are only possible for SVE vectors");
15351 Offset.getValueType().isVector())
15362 if (!isValidImmForSVEVecImmAddrMode(
Offset,
15384 if (!OnlyPackedOffsets &&
15406 if (RetVT.
isInteger() && (RetVT != HwRetVt))
15444 "Sign extending from an invalid type");
15451 return DAG.
getNode(SOpc,
DL,
N->getValueType(0), Ext);
15463 unsigned MemVTOpNum = 4;
15529 if ((SignExtSrcVT != SrcMemVT) || !Src.
hasOneUse())
15532 EVT DstVT =
N->getValueType(0);
15551 const unsigned OffsetPos = 4;
15574 unsigned ScalarSizeInBytes) {
15575 const unsigned ImmPos = 4, OffsetPos = 3;
15577 if (isValidImmForSVEVecImmAddrMode(
N->getOperand(ImmPos), ScalarSizeInBytes))
15595 switch (
N->getOpcode()) {
15629 return performSignExtendInRegCombine(
N, DCI, DAG);
15635 return performSelectCombine(
N, DCI);
15637 return performVSelectCombine(
N, DCI.
DAG);
15639 if (performTBISimplification(
N->getOperand(1), DCI, DAG))
15643 return performSTORECombine(
N, DCI, DAG, Subtarget);
15646 return performMaskedGatherScatterCombine(
N, DCI, DAG);
15648 return performBRCONDCombine(
N, DCI, DAG);
15651 return performTBZCombine(
N, DCI, DAG);
15653 return performCONDCombine(
N, DCI, DAG, 2, 3);
15657 return performNVCASTCombine(
N);
15669 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
15670 return combineSVEPrefetchVecBaseImmOff(
N, DAG, 1 );
15671 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
15672 return combineSVEPrefetchVecBaseImmOff(
N, DAG, 2 );
15673 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
15674 return combineSVEPrefetchVecBaseImmOff(
N, DAG, 4 );
15675 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
15676 return combineSVEPrefetchVecBaseImmOff(
N, DAG, 8 );
15677 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
15678 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
15679 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
15680 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
15681 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
15682 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
15683 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
15684 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
15685 return legalizeSVEGatherPrefetchOffsVec(
N, DAG);
15686 case Intrinsic::aarch64_neon_ld2:
15687 case Intrinsic::aarch64_neon_ld3:
15688 case Intrinsic::aarch64_neon_ld4:
15689 case Intrinsic::aarch64_neon_ld1x2:
15690 case Intrinsic::aarch64_neon_ld1x3:
15691 case Intrinsic::aarch64_neon_ld1x4:
15692 case Intrinsic::aarch64_neon_ld2lane:
15693 case Intrinsic::aarch64_neon_ld3lane:
15694 case Intrinsic::aarch64_neon_ld4lane:
15695 case Intrinsic::aarch64_neon_ld2r:
15696 case Intrinsic::aarch64_neon_ld3r:
15697 case Intrinsic::aarch64_neon_ld4r:
15698 case Intrinsic::aarch64_neon_st2:
15699 case Intrinsic::aarch64_neon_st3:
15700 case Intrinsic::aarch64_neon_st4:
15701 case Intrinsic::aarch64_neon_st1x2:
15702 case Intrinsic::aarch64_neon_st1x3:
15703 case Intrinsic::aarch64_neon_st1x4:
15704 case Intrinsic::aarch64_neon_st2lane:
15705 case Intrinsic::aarch64_neon_st3lane:
15706 case Intrinsic::aarch64_neon_st4lane:
15707 return performNEONPostLDSTCombine(
N, DCI, DAG);
15708 case Intrinsic::aarch64_sve_ldnt1:
15710 case Intrinsic::aarch64_sve_ld1rq:
15712 case Intrinsic::aarch64_sve_ld1ro:
15714 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
15716 case Intrinsic::aarch64_sve_ldnt1_gather:
15718 case Intrinsic::aarch64_sve_ldnt1_gather_index:
15719 return performGatherLoadCombine(
N, DAG,
15721 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
15723 case Intrinsic::aarch64_sve_ld1:
15725 case Intrinsic::aarch64_sve_ldnf1:
15727 case Intrinsic::aarch64_sve_ldff1:
15729 case Intrinsic::aarch64_sve_st1:
15731 case Intrinsic::aarch64_sve_stnt1:
15733 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
15735 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
15737 case Intrinsic::aarch64_sve_stnt1_scatter:
15739 case Intrinsic::aarch64_sve_stnt1_scatter_index:
15741 case Intrinsic::aarch64_sve_ld1_gather:
15743 case Intrinsic::aarch64_sve_ld1_gather_index:
15744 return performGatherLoadCombine(
N, DAG,
15746 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
15749 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
15752 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
15753 return performGatherLoadCombine(
N, DAG,
15756 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
15757 return performGatherLoadCombine(
N, DAG,
15760 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
15762 case Intrinsic::aarch64_sve_ldff1_gather:
15764 case Intrinsic::aarch64_sve_ldff1_gather_index:
15765 return performGatherLoadCombine(
N, DAG,
15767 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
15768 return performGatherLoadCombine(
N, DAG,
15771 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
15772 return performGatherLoadCombine(
N, DAG,
15775 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
15776 return performGatherLoadCombine(
N, DAG,
15779 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
15780 return performGatherLoadCombine(
N, DAG,
15783 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
15784 return performGatherLoadCombine(
N, DAG,
15786 case Intrinsic::aarch64_sve_st1_scatter:
15788 case Intrinsic::aarch64_sve_st1_scatter_index:
15790 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
15793 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
15796 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
15797 return performScatterStoreCombine(
N, DAG,
15800 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
15801 return performScatterStoreCombine(
N, DAG,
15804 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
15806 case Intrinsic::aarch64_sve_tuple_get: {
15813 EVT ResVT =
N->getValueType(0);
15820 case Intrinsic::aarch64_sve_tuple_set: {
15827 EVT TupleVT = Tuple.getValueType();
15831 uint64_t NumLanes =
15834 if ((TupleLanes % NumLanes) != 0)
15837 uint64_t NumVecs = TupleLanes / NumLanes;
15840 for (
unsigned I = 0;
I < NumVecs; ++
I) {
15853 case Intrinsic::aarch64_sve_tuple_create2:
15854 case Intrinsic::aarch64_sve_tuple_create3:
15855 case Intrinsic::aarch64_sve_tuple_create4: {
15860 for (
unsigned I = 2;
I <
N->getNumOperands(); ++
I)
15863 EVT VT = Opnds[0].getValueType();
15867 (
N->getNumOperands() - 2));
15871 case Intrinsic::aarch64_sve_ld2:
15872 case Intrinsic::aarch64_sve_ld3:
15873 case Intrinsic::aarch64_sve_ld4: {
15877 SDValue BasePtr =
N->getOperand(3);
15878 SDValue LoadOps[] = {Chain, Mask, BasePtr};
15879 unsigned IntrinsicID =
15882 LowerSVEStructLoad(IntrinsicID, LoadOps,
N->
getValueType(0), DAG,
DL);
15899bool AArch64TargetLowering::isUsedByReturnOnly(
SDNode *
N,
15901 if (
N->getNumValues() != 1)
15903 if (!
N->hasNUsesOfValue(1, 0))
15907 SDNode *Copy = *
N->use_begin();
15911 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
15914 TCChain = Copy->getOperand(0);
15918 bool HasRet =
false;
15936bool AArch64TargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
15940bool AArch64TargetLowering::getIndexedAddressParts(
SDNode *Op,
SDValue &Base,
15948 Base =
Op->getOperand(0);
15954 RHSC = -(uint64_t)RHSC;
15964bool AArch64TargetLowering::getPreIndexedAddressParts(
SDNode *
N,
SDValue &Base,
15971 VT =
LD->getMemoryVT();
15972 Ptr =
LD->getBasePtr();
15974 VT =
ST->getMemoryVT();
15975 Ptr =
ST->getBasePtr();
15980 if (!getIndexedAddressParts(Ptr.
getNode(), Base,
Offset, AM, IsInc, DAG))
15986bool AArch64TargetLowering::getPostIndexedAddressParts(
15992 VT =
LD->getMemoryVT();
15993 Ptr =
LD->getBasePtr();
15995 VT =
ST->getMemoryVT();
15996 Ptr =
ST->getBasePtr();
16001 if (!getIndexedAddressParts(Op, Base,
Offset, AM, IsInc, DAG))
16032 unsigned AcrossOp) {
16049 return std::make_pair(Lo, Hi);
16052void AArch64TargetLowering::ReplaceExtractSubVectorResults(
16055 EVT InVT =
In.getValueType();
16062 EVT VT =
N->getValueType(0);
16075 unsigned Index = CIndex->getZExtValue();
16088 SDLoc dl(V.getNode());
16099 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
16109 "AtomicCmpSwap on types less than 128 should be legal");
16124 switch (
MemOp->getOrdering()) {
16126 Opcode = AArch64::CASPX;
16129 Opcode = AArch64::CASPAX;
16132 Opcode = AArch64::CASPLX;
16136 Opcode = AArch64::CASPALX;
16146 unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
16161 SDValue Ops[] = {
N->getOperand(1), Desired.first, Desired.second,
16162 New.first, New.second,
N->getOperand(0)};
16164 AArch64::CMP_SWAP_128,
SDLoc(
N),
16175void AArch64TargetLowering::ReplaceNodeResults(
16177 switch (
N->getOpcode()) {
16215 assert(
N->getValueType(0) ==
MVT::i128 &&
"unexpected illegal conversion");
16223 "unexpected load's value type");
16233 DAG.
getVTList({MVT::i64, MVT::i64, MVT::Other}),
16234 {LoadNode->getChain(), LoadNode->getBasePtr()}, LoadNode->
getMemoryVT(),
16243 ReplaceExtractSubVectorResults(
N,
Results, DAG);
16246 EVT VT =
N->getValueType(0);
16248 "custom lowering for unexpected type");
16255 case Intrinsic::aarch64_sve_clasta_n: {
16259 N->getOperand(1), Op2,
N->getOperand(3));
16263 case Intrinsic::aarch64_sve_clastb_n: {
16267 N->getOperand(1), Op2,
N->getOperand(3));
16271 case Intrinsic::aarch64_sve_lasta: {
16274 N->getOperand(1),
N->getOperand(2));
16278 case Intrinsic::aarch64_sve_lastb: {
16281 N->getOperand(1),
N->getOperand(2));
16296unsigned AArch64TargetLowering::combineRepeatedFPDivisors()
const {
16317 unsigned Size =
SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
16318 return Size == 128;
16342 if (Subtarget->
hasLSE())
16389 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16396 if (ValTy->getPrimitiveSizeInBits() == 128) {
16398 IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
16402 Value *LoHi = Builder.CreateCall(Ldxr,
Addr,
"lohi");
16404 Value *Lo = Builder.CreateExtractValue(LoHi, 0,
"lo");
16405 Value *Hi = Builder.CreateExtractValue(LoHi, 1,
"hi");
16406 Lo = Builder.CreateZExt(Lo, ValTy,
"lo64");
16407 Hi = Builder.CreateZExt(Hi, ValTy,
"hi64");
16408 return Builder.CreateOr(
16412 Type *Tys[] = {
Addr->getType() };
16414 IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
16421 Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr,
Addr), IntEltTy);
16423 return Builder.CreateBitCast(Trunc, EltTy);
16428 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16435 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16443 IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
16447 Value *Lo = Builder.CreateTrunc(Val, Int64Ty,
"lo");
16448 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty,
"hi");
16450 return Builder.CreateCall(Stxr, {Lo, Hi,
Addr});
16454 IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
16455 Type *Tys[] = {
Addr->getType() };
16460 Val = Builder.CreateBitCast(Val, IntValTy);
16462 return Builder.CreateCall(Stxr,
16463 {Builder.CreateZExtOrBitCast(
16469 Type *Ty, CallingConv::ID CallConv,
bool isVarArg)
const {
16470 if (Ty->isArrayTy())
16473 const TypeSize &TySize = Ty->getPrimitiveSizeInBits();
16480bool AArch64TargetLowering::shouldNormalizeToSelectSequence(
LLVMContext &,
16514 M.getOrInsertGlobal(
"__security_cookie",
16523 F->addAttribute(1, Attribute::AttrKind::InReg);
16533 return M.getGlobalVariable(
"__security_cookie");
16569 return Mask->getValue().isPowerOf2();
16575 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
16579 X, XC, CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG))
16582 return X.getValueType().isScalarInteger() || NewShiftOpcode ==
ISD::SHL;
16603 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
16612 if (AArch64::GPR64RegClass.
contains(*
I))
16613 RC = &AArch64::GPR64RegClass;
16614 else if (AArch64::FPR64RegClass.
contains(*
I))
16615 RC = &AArch64::FPR64RegClass;
16625 assert(Entry->getParent()->getFunction().hasFnAttribute(
16626 Attribute::NoUnwind) &&
16627 "Function should be nounwind in insertCopiesSplitCSR!");
16628 Entry->addLiveIn(*
I);
16633 for (
auto *Exit : Exits)
16635 TII->get(TargetOpcode::COPY), *
I)
16669void AArch64TargetLowering::finalizeLowering(
MachineFunction &MF)
const {
16679bool AArch64TargetLowering::shouldLocalize(
16681 switch (
MI.getOpcode()) {
16682 case TargetOpcode::G_GLOBAL_VALUE: {
16693 case AArch64::ADRP:
16694 case AArch64::G_ADD_LOW:
16722 "Expected legal fixed length vector!");
16748 "Expected legal fixed length vector!");
16755 PgPattern = AArch64SVEPredPattern::vl1;
16758 PgPattern = AArch64SVEPredPattern::vl2;
16761 PgPattern = AArch64SVEPredPattern::vl4;
16764 PgPattern = AArch64SVEPredPattern::vl8;
16767 PgPattern = AArch64SVEPredPattern::vl16;
16770 PgPattern = AArch64SVEPredPattern::vl32;
16773 PgPattern = AArch64SVEPredPattern::vl64;
16776 PgPattern = AArch64SVEPredPattern::vl128;
16779 PgPattern = AArch64SVEPredPattern::vl256;
16815 "Expected legal scalable vector!");
16817 return getPTrue(DAG,
DL, PredTy, AArch64SVEPredPattern::all);
16830 "Expected to convert into a scalable vector!");
16831 assert(V.getValueType().isFixedLengthVector() &&
16832 "Expected a fixed length vector operand!");
16841 "Expected to convert into a fixed length vector!");
16842 assert(V.getValueType().isScalableVector() &&
16843 "Expected a scalable vector operand!");
16850SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
16855 EVT VT =
Op.getValueType();
16859 ContainerVT,
DL,
Load->getChain(),
Load->getBasePtr(),
Load->getOffset(),
16861 Load->getMemoryVT(),
Load->getMemOperand(),
Load->getAddressingMode(),
16862 Load->getExtensionType());
16870SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
16875 EVT VT =
Store->getValue().getValueType();
16882 Store->getMemOperand(),
Store->getAddressingMode(),
16883 Store->isTruncatingStore());
16886SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
16889 EVT VT =
Op.getValueType();
16897 return LowerToPredicatedOp(Op, DAG, PredOpcode,
true);
16912 SDValue Op0Lo = DAG.
getNode(UnpkLo, dl, ScalableWidenedVT, Op0);
16913 SDValue Op1Lo = DAG.
getNode(UnpkLo, dl, ScalableWidenedVT, Op1);
16914 SDValue Op0Hi = DAG.
getNode(UnpkHi, dl, ScalableWidenedVT, Op0);
16915 SDValue Op1Hi = DAG.
getNode(UnpkHi, dl, ScalableWidenedVT, Op1);
16931 ResultLo, ResultHi);
16936SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
16938 EVT VT =
Op.getValueType();
16972SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
16974 EVT VT =
Op.getValueType();
17014 bool OverrideNEON)
const {
17015 EVT VT =
Op.getValueType();
17019 if (useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
17024 for (
const SDValue &V :
Op->op_values()) {
17037 assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
17038 "Only fixed length vectors are supported!");
17052 for (
const SDValue &V :
Op->op_values()) {
17053 assert((!V.getValueType().isVector() ||
17054 V.getValueType().isScalableVector()) &&
17055 "Only scalable vectors are supported!");
17070 EVT VT =
Op.getValueType();
17071 assert(useSVEForFixedLengthVectorVT(VT) &&
17072 "Only expected to lower fixed length vector operation!");
17077 for (
const SDValue &V :
Op->op_values()) {
17081 if (!V.getValueType().isVector()) {
17087 assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
17088 "Only fixed length vectors are supported!");
17096SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(
SDValue ScalarOp,
17104 EVT ContainerVT = SrcVT;
17115 DAG.
getUNDEF(ContainerVT), AccOp, Zero);
17124SDValue AArch64TargetLowering::LowerPredReductionToSVE(
SDValue ReduceOp,
17128 EVT OpVT =
Op.getValueType();
17157SDValue AArch64TargetLowering::LowerReductionToSVE(
unsigned Opcode,
17164 if (useSVEForFixedLengthVectorVT(SrcVT,
true)) {
17171 SrcVT.getVectorElementType();
17189AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(
SDValue Op,
17191 EVT VT =
Op.getValueType();
17194 EVT InVT =
Op.getOperand(1).getValueType();
17201 EVT MaskVT =
Op.getOperand(0).getValueType();
17213SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
17216 EVT InVT =
Op.getOperand(0).getValueType();
17219 assert(useSVEForFixedLengthVectorVT(InVT) &&
17220 "Only expected to lower fixed length vector operation!");
17222 "Expected integer result of the same bit length as the inputs!");
17234 {Pg, Op1, Op2,
Op.getOperand(2)});
17244 EVT InVT =
Op.getValueType();
17250 "Only expect to cast between legal scalable vector types!");
17253 "Cannot cast between data and predicate scalable vector types!");
17263 assert((VT == PackedVT || InVT == PackedInVT) &&
17264 "Cannot cast between unpacked scalable vector types!");
17267 if (InVT != PackedInVT)
17273 if (VT != PackedVT)
unsigned const MachineRegisterInfo * MRI
static unsigned MatchRegisterName(StringRef Name)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isConcatMask(ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector...
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isSingletonEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue NormalizeBuildVector(SDValue Op, SelectionDAG &DAG)
static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
static SDValue GenerateTBL(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static std::pair< SDValue, SDValue > splitInt128(SDValue N, SelectionDAG &DAG)
static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo)
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
static bool isLegalArithImmed(uint64_t C)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG)
unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static bool areExtractShuffleVectors(Value *Op1, Value *Op2)
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector ...
unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static bool isREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
static SDValue performSRLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point divide by power of two into fixed-point to floating-point conversion.
static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG)
static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MVT getSVEContainerType(EVT ContentTy)
static bool isMergePassthruOpcode(unsigned Opc)
static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG)
Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
static void ReplaceCMP_SWAP_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static bool canGuaranteeTCO(CallingConv::ID CC)
Return true if the calling convention is one that we can guarantee TCO for.
static SDValue performABSCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static cl::opt< bool > EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))
static bool isUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v,...
static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue convertMergedOpToPredOp(SDNode *N, unsigned PredOpc, SelectionDAG &DAG)
static unsigned getDUPLANEOp(EVT EltType)
static void changeFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static bool isTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI)
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG)
static EVT getPackedSVEVectorVT(EVT VT)
static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG)
static void ReplaceReductionResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi)
An EXTR instruction is made up of two shifts, ORed together.
static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT, unsigned Opcode, SelectionDAG &DAG)
static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N, SelectionDAG &DAG)
static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isINSMask(ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG)
static unsigned getIntrinsicID(const SDNode *N)
static bool IsSVECntIntrinsic(SDValue S)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool canEmitConjunction(const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
static bool isWideDUPMask(ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp)
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vect...
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, bool isSigned)
static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG)
static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG)
static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc)
static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16)
static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG)
static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
unsigned getSignExtendedGatherOpcode(unsigned Opcode)
static SDValue performAddSubCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG)
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC)
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index, EVT MemVT, unsigned &Opcode, bool IsGather, SelectionDAG &DAG)
static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static unsigned getCmpOperandFoldingProfit(SDValue Op)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static SDValue performConcatVectorsCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG)
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
SDLoc DL(N)
Simplify Addr given that the top byte of it is ignored by HW during / address translation.
static SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern)
static bool isEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
static SDValue tryCombineFixedPointConvert(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG)
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZE...
static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
static EVT getPromotedVTForPredicate(EVT VT)
static void changeFPCCToANDAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert a DAG fp condition code to an AArch64 CC.
static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X,...
static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG)
static bool isAllConstantBuildVector(const SDValue &PotentialBVec, uint64_t &ConstVal)
static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG)
static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond)
static bool isSetCCOrZExtSetCC(const SDValue &Op, SetCCInfoAndKind &Info)
cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG)
static SDValue performPostLD1Combine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2)
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
std::pair< SDValue, uint64_t > lookThroughSignExtension(SDValue Val)
static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG)
static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static PredicateConstraint parsePredicateConstraint(StringRef Constraint)
static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isPackedVectorType(EVT VT, SelectionDAG &DAG)
Returns true if VT's elements occupy the lowest bit positions of its associated register class withou...
static SDValue performAddSubLongCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static bool isTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v,...
static bool isZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static SDValue performExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl, SelectionDAG &DAG, SDValue Chain, bool IsSignaling)
static bool isSignExtended(SDNode *N, SelectionDAG &DAG)
static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG)
bool getGatherScatterIndexIsExtended(SDValue Index)
static bool isZeroExtended(SDNode *N, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool isCMN(SDValue Op, ISD::CondCode CC)
static SDValue tryCombineToEXTR(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low...
static bool isOperandOfVmullHighP64(Value *Op)
Check if Op could be used with vmull_high_p64 intrinsic.
static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static bool isEssentiallyExtractHighSubvector(SDValue N)
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
static Value * UseTlsOffset(IRBuilder<> &IRB, unsigned Offset)
static unsigned getExtFactor(SDValue &V)
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vect...
static SDValue performExtractVectorEltCombine(SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG)
static const MVT MVT_CC
Value type used for condition codes.
static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle, SelectionDAG &DAG)
Combines a dup(sext/zext) node pattern into sext/zext(dup) making use of the vector SExt/ZExt rather ...
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl)
static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static cl::opt< bool > EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true))
static bool isZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v,...
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static std::pair< SDValue, SDValue > getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
#define FALKOR_STRIDED_ACCESS_MD
static const unsigned PerfectShuffleTable[6561+1]
MachineBasicBlock MachineBasicBlock::iterator MBBI
static bool isConstant(const MachineInstr &MI)
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static const MCPhysReg GPRArgRegs[]
Function Alias Analysis Results
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define STATISTIC(VARNAME, DESC)
static const int BlockSize
This defines the Use class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static constexpr int Concat[]
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
unsigned getVarArgsFPRSize() const
void setVarArgsStackIndex(int Index)
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
void setIsSplitCSR(bool s)
int getVarArgsFPRIndex() const
void incNumLocalDynamicTLSAccesses()
void setBytesInStackArgArea(unsigned bytes)
int getVarArgsStackIndex() const
void setVarArgsGPRIndex(int Index)
int getVarArgsGPRIndex() const
void setVarArgsFPRSize(unsigned Size)
unsigned getVarArgsGPRSize() const
unsigned getSRetReturnReg() const
void setSRetReturnReg(unsigned Reg)
unsigned getBytesInStackArgArea() const
void setJumpTableEntryInfo(int Idx, unsigned Size, MCSymbol *PCRelSym)
void setVarArgsFPRIndex(int Index)
void setVarArgsGPRSize(unsigned Size)
void setArgumentStackToRestore(unsigned bytes)
void UpdateCustomCalleeSavedRegs(MachineFunction &MF) const
static bool hasSVEArgsOrReturn(const MachineFunction *MF)
bool isTargetWindows() const
bool hasFuseLiterals() const
unsigned getPrefLoopLogAlignment() const
const AArch64RegisterInfo * getRegisterInfo() const override
unsigned getPrefFunctionLogAlignment() const
bool isMisaligned128StoreSlow() const
const AArch64InstrInfo * getInstrInfo() const override
unsigned getMaximumJumpTableSize() const
bool isTargetDarwin() const
bool isTargetILP32() const
ARMProcFamilyEnum getProcFamily() const
Returns ARM processor family.
unsigned classifyGlobalFunctionReference(const GlobalValue *GV, const TargetMachine &TM) const
bool isTargetMachO() const
bool supportsAddressTopByteIgnored() const
CPU has TBI (top byte of addresses is ignored during HW address translation) and OS enables it.
bool isTargetAndroid() const
const Triple & getTargetTriple() const
bool isCallingConvWin64(CallingConv::ID CC) const
unsigned getMinSVEVectorSizeInBits() const
bool outlineAtomics() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
bool isLittleEndian() const
bool hasAggressiveFMA() const
bool isXRegisterReserved(size_t i) const
bool requiresStrictAlign() const
bool isTargetFuchsia() const
bool predictableSelectIsExpensive() const
bool useSVEForFixedLengthVectors() const
bool hasCustomCallingConv() const
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
unsigned getVaListSizeInBits(const DataLayout &DL) const override
Returns the size of the platform's va_list object.
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns true if the given (atomic) store should be expanded by the IR-level AtomicExpand pass into an...
int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS) const override
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
Value * emitLoadLinked(IRBuilder<> &Builder, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const override
For some targets, an LLVM struct type must be broken down into multiple simple types,...
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ISD::SETCC ValueType.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalICmpImmediate(int64_t) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a ldN intrinsic.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
bool fallBackToDAGISel(const Instruction &Inst) const override
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
MachineBasicBlock * EmitLoweredCatchRet(MachineInstr &MI, MachineBasicBlock *BB) const
Value * getIRStackGuard(IRBuilder<> &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isProfitableToHoist(Instruction *I) const override
Check if it is profitable to hoist instruction in then/else to if.
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if it is profitable to reduce a load to a smaller type.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a stN intrinsic.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
Value * getSafeStackPointerLocation(IRBuilder<> &IRB) const override
If the target has a standard location for the unsafe stack pointer, returns the address of that locat...
MachineBasicBlock * EmitF128CSEL(MachineInstr &MI, MachineBasicBlock *BB) const
LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &FuncAttributes) const override
LLT returning variant.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool needsFixedCatchObjects() const override
Used for exception handling on Win64.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
AArch64TargetLowering(const TargetMachine &TM, const AArch64Subtarget &STI)
bool isLegalAddImmediate(int64_t) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool shouldConsiderGEPOffsetSplit() const override
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Returns false if N is a bit extraction pattern of (X >> C) & Mask.
bool enableAggressiveFMAFusion(EVT VT) const override
Enable aggressive FMA fusion on targets that want it.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, unsigned Align=1, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
Value * emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
bool mergeStoresAfterLegalization(EVT VT) const override
SVE code generation for fixed length vectors does not custom lower BUILD_VECTOR.
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
APInt sextOrSelf(unsigned width) const
Sign extend or truncate to width.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned logBase2() const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
int64_t getSExtValue() const
Get sign extended value.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
bool isFloatingPointOperation() const
BinOp getOperation() const
This is an SDNode representing atomic operations.
This class holds the attributes for a function, its return value, and its parameters.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Equivalent to hasAttribute(AttributeList::FunctionIndex, Kind) but may be faster.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const BlockAddress * getBlockAddress() const
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
CCValAssign - Represent assignment of one arg/retval to a location.
unsigned getLocMemOffset() const
Register getLocReg() const
LocInfo getLocInfo() const
Value * getArgOperand(unsigned i) const
unsigned getNumArgOperands() const
This class represents a function call, abstracting a target machine's calling convention.
bool isZero() const
Return true if the value is positive or negative zero.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
uint64_t getZExtValue() const
int64_t getSExtValue() const
bool isAllOnesValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
const Function & getFunction() const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
int64_t getOffset() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
BasicBlock * GetInsertBlock() const
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Value * CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
This is an important class for using LLVM in a threaded context.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
static ElementCount getScalable(ScalarTy MinVal)
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static mvt_range fixedlen_vector_valuetypes()
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static mvt_range integer_valuetypes()
static mvt_range integer_fixedlen_vector_valuetypes()
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static mvt_range fp_fixedlen_vector_valuetypes()
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static mvt_range all_valuetypes()
SimpleValueType Iteration.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static mvt_range fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setStackID(int ObjectIdx, uint8_t ID)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
void setReturnAddressIsTaken(bool s)
void computeMaxCallFrameSize(const MachineFunction &MF)
Computes the maximum size of a callframe and the AdjustsStack property.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineInstr - Allocate a new MachineInstr.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
This is a base class used to represent MGATHER and MSCATTER nodes.
const SDValue & getIndex() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
bool isIndexSigned() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Class to represent pointers.
Type * getElementType() const
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const TargetSubtargetInfo & getSubtarget() const
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offset=0, unsigned TargetFlags=0)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplatValue(SDValue V)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
bool isZeroEltSplat() const
Return true if all elements of this shuffle are the same value as the first element of exactly one so...
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isExtractSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &Index)
Return true if this shuffle mask is an extract subvector mask.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
LLVM_NODISCARD size_t size() const
size - Get the string size.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setTargetDAGCombine(ISD::NodeType NT)
Targets should invoke this method for each target independent node that they want to provide a custom...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual Value * getSafeStackPointerLocation(IRBuilder<> &IRB) const
Returns the target-specific address of the unsafe stack pointer.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilder<> &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool EnableExtLdPromotion
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned EmitCallSiteInfo
The flag enables call site info production.
unsigned TLSSize
Bit size of immediate TLS offsets (0 == use the default).
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Triple - Helper class for working with autoconf configuration names.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ScalarTy getFixedSize() const
ScalarTy getKnownMinSize() const
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt64Ty(LLVMContext &C)
@ FloatTyID
32-bit floating point type
@ DoubleTyID
64-bit floating point type
static Type * getVoidTy(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
ScalarTy getValue() const
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Base class of all SIMD vector types.
Type * getElementType() const
Type * getIndexedType() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
@ GLDFF1S_SXTW_MERGE_ZERO
@ GLDFF1_SCALED_MERGE_ZERO
@ GLD1_SXTW_SCALED_MERGE_ZERO
@ FP_EXTEND_MERGE_PASSTHRU
@ FP_ROUND_MERGE_PASSTHRU
@ GLDFF1_SXTW_SCALED_MERGE_ZERO
@ UINT_TO_FP_MERGE_PASSTHRU
@ FROUNDEVEN_MERGE_PASSTHRU
@ GLD1S_UXTW_SCALED_MERGE_ZERO
@ GLDNT1_INDEX_MERGE_ZERO
@ GLDFF1_UXTW_SCALED_MERGE_ZERO
@ FNEARBYINT_MERGE_PASSTHRU
@ GLDFF1S_SCALED_MERGE_ZERO
@ GLDFF1S_UXTW_SCALED_MERGE_ZERO
@ ZERO_EXTEND_INREG_MERGE_PASSTHRU
@ NVCAST
Natural vector cast.
@ BITREVERSE_MERGE_PASSTHRU
@ GLDFF1S_UXTW_MERGE_ZERO
@ SIGN_EXTEND_INREG_MERGE_PASSTHRU
@ GLDFF1S_SXTW_SCALED_MERGE_ZERO
@ GLD1S_SCALED_MERGE_ZERO
@ SINT_TO_FP_MERGE_PASSTHRU
@ GLD1_UXTW_SCALED_MERGE_ZERO
@ GLD1S_SXTW_SCALED_MERGE_ZERO
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint8_t encodeAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType9(uint64_t Imm)
static bool isAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType5(uint64_t Imm)
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType10(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType9(uint64_t Imm)
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static bool isAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType5(uint64_t Imm)
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static bool isAdvSIMDModImmType10(uint64_t Imm)
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType8(uint64_t Imm)
static bool isAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType11(uint64_t Imm)
static bool isAdvSIMDModImmType11(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType6(uint64_t Imm)
static bool isAdvSIMDModImmType8(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType6(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType1(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType1(uint64_t Imm)
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static constexpr unsigned SVEBitsPerBlock
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AArch64_SVE_VectorCall
Calling convention between AArch64 SVE functions.
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
C - The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ FLT_ROUNDS_
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isOverflowIntrOpRes(SDValue Op)
Returns true if the specified value is the overflow result from one of the overflow intrinsic nodes.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
bool match(Val *V, const Pattern &P)
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
class_match< UndefValue > m_Undef()
Match an arbitrary undef constant.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_or< CastClass_match< OpTy, Instruction::ZExt >, CastClass_match< OpTy, Instruction::SExt > > m_ZExtOrSExt(const OpTy &Op)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
CodeModel::Model getCodeModel()
DiagnosticInfoOptimizationBase::Argument NV
This class represents lattice values for constants.
bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
LLVM_NODISCARD std::enable_if_t< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > dyn_cast_or_null(const Y &Val)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(uint64_t V1, const APInt &V2)
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
testing::Matcher< const detail::ErrorHolder & > Failed()
bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
constexpr bool isUInt< 32 >(uint64_t x)
void shuffle(Iterator first, Iterator last, RNG &&g)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr std::enable_if_t<(N< 64), bool > isUInt(uint64_t X)
Checks if an unsigned integer fits into the given bit width.
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
LLVM_NODISCARD std::enable_if_t< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > dyn_cast(const Y &Val)
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
LLVM_NODISCARD bool isa(const Y &Val)
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Z
zlib style complession
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
@ Invalid
Denotes invalid value.
std::enable_if_t<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > cast(const Y &Val)
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool isAcquireOrStronger(AtomicOrdering AO)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr unsigned BitWidth
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
gep_type_iterator gep_type_begin(const User *GEP)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool CC_AArch64_DarwinPCS_ILP32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
static const MachineMemOperand::Flags MOStridedAccess
bool CC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper structure to keep track of a SET_CC lowered into AArch64 code.
Helper structure to keep track of ISD::SET_CC operands.
Helper structure to be able to read SetCC information.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
EVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
bool isFixedLengthVector() const
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Describes a register that needs to be forwarded from the prologue to a musttail call.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static KnownBits commonBits(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits common to LHS and RHS.
unsigned getBitWidth() const
Get the bit width of this value.
Structure used to represent pair of argument number after call lowering and register used to transfer...
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op is the value the constraint a...
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowReassociation(bool b)
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
This structure contains all information that is necessary for lowering calls.
bool isBeforeLegalizeOps() const
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)
Helper structure to keep track of SetCC information.